Пример #1
0
        /// <summary>
        /// Creates a singleton settings object and loads parameters from file, if it exists.
        /// </summary>
        public static void GlobalSettingsInit()
        {
            Settings    = new GlobalSettings();
            appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
            settingsFN  = Path.Combine(appDataPath, "furdown\\furdown.conf");
            // true portable mode
            if (File.Exists("./furdown-portable.conf"))
            {
                settingsFN = "./furdown-portable.conf";
            }
            // if settings file exists, load it
            bool needToSetDefaults = false;

            if (File.Exists(settingsFN))
            {
                using (Stream stream = File.Open(settingsFN, FileMode.Open))
                {
                    var bformatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
                    try
                    {
                        Settings = (GlobalSettings)bformatter.Deserialize(stream);
                    }
                    // settings file is present, but failed to be loaded
                    catch
                    {
                        needToSetDefaults = true;
                        Console.WriteLine("Settings file is incompatible or corrupted, restoring defaults.");
                    }
                }
            }
            else
            {
                needToSetDefaults = true;
            }
            // set defaults, if settings file does not exist or otherwise not loaded
            if (needToSetDefaults)
            {
                Settings.downloadPath          = Path.Combine(appDataPath, "furdown\\downloads");
                Settings.systemPath            = Path.Combine(appDataPath, "furdown\\system");
                Settings.filenameTemplate      = "%ARTIST%\\%SUBMID%.%FILEPART%";
                Settings.descrFilenameTemplate = "%ARTIST%\\%SUBMID%.%FILEPART%.dsc.htm";
                Settings.downloadOnlyOnce      = true;
                try {
                    Directory.CreateDirectory(Settings.downloadPath);
                    Directory.CreateDirectory(Settings.systemPath);
                }
                catch
                {
                    Console.WriteLine("[Error] Default paths are invalid!");
                }
            }
            SubmissionsDB.Load();
        }
Пример #2
0
 /// <summary>
 /// Saves settings to file.
 /// </summary>
 public static void GlobalSettingsSave()
 {
     try
     {
         Directory.CreateDirectory(Path.GetDirectoryName(settingsFN));
         using (Stream stream = File.Open(settingsFN, FileMode.Create))
         {
             var bformatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
             bformatter.Serialize(stream, Settings);
         }
         SubmissionsDB.Save();
     }
     catch
     {
         Console.WriteLine("[Error] Failed to write settings file!");
         System.Windows.Forms.MessageBox.Show("Failed to write settings file!");
     }
 }
Пример #3
0
        private void removeIdsFromDb_Click(object sender, EventArgs e)
        {
            int counter = 0;

            string[] lines = dbSubmTextBox.Lines;
            dbSubmTextBox.Text = "";
            for (int i = 0; i < lines.Count(); i++)
            {
                try
                {
                    if (SubmissionsDB.DB.RemoveSubmission(uint.Parse(lines[i])))
                    {
                        counter++;
                    }
                    else
                    {
                        dbSubmTextBox.Text += lines[i] + Environment.NewLine;
                    }
                }
                catch { } // not an integer - do not care
            }
            SubmissionsDB.Save();
            MessageBox.Show(counter.ToString() + " submissions have been removed.");
        }
Пример #4
0
 public static void Load()
 {
     try
     {
         string dbFN = Path.Combine(GlobalSettings.Settings.systemPath, "furdown.db");
         using (Stream stream = File.Open(dbFN, FileMode.Open))
         {
             var bformatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
             try
             {
                 DB = (SubmissionsDB)bformatter.Deserialize(stream);
             }
             // DB file is present, but failed to be loaded
             catch
             {
                 Console.WriteLine("Failed to load DB, using empty one.");
             }
         }
     }
     catch
     {
         Console.WriteLine("No DB found.");
     }
 }
Пример #5
0
        public async Task <ProcessingResults> ProcessSubmissionsList(List <string> subs, bool needDescription)
        {
            Console.WriteLine("Processing submissions list...");
            ProcessingResults res = new ProcessingResults();

            // iterate over all the submissions in list
            for (int i = subs.Count - 1; i >= 0; i--)
            {
                string subId = subs[i];
                // don't care about empty strings
                if (string.IsNullOrEmpty(subId))
                {
                    continue;
                }
                Console.WriteLine("> Processing submission #" + subId);
                // check if in DB already
                try
                {
                    if (SubmissionsDB.DB.Exists(uint.Parse(subId)) &&
                        GlobalSettings.Settings.downloadOnlyOnce)
                    {
                        Console.WriteLine("Skipped (present in DB)");
                        continue;
                    }
                }
                catch
                {
                    Console.WriteLine("Unexpected error (DB presence check failed)!");
                    continue;
                }
                string subUrl = "https://www.furaffinity.net/view/" + subId;

                // get submission page
                int    attempts = 3;
                string cpage    = "";
beforeawait:
                try
                {
                    Console.WriteLine("Getting page: " + subUrl);
                    cpage = await http.GetStringAsync(subUrl);
                }
                catch (Exception E)
                {
                    Console.WriteLine("GET request error (" + subUrl + "): " + E.Message);
                    attempts--;
                    System.Threading.Thread.Sleep(2000);
                    if (attempts > 0)
                    {
                        goto beforeawait;
                    }
                    else
                    {
                        Console.WriteLine("Giving up on #" + subId);
                        res.failedToGetPage.Add(subId);
                        continue;
                    }
                }

                // process submission page
                string          downbtnkey = "<a href=\"//d.facdn.net/";
                SubmissionProps sp         = new SubmissionProps();
                sp.SUBMID = subId;
                int keypos = cpage.IndexOf(downbtnkey, StringComparison.Ordinal);
                if (keypos < 0)
                {
                    Console.WriteLine("[Warning] got page, but it doesn't contain any download links.");
                    res.failedToGetPage.Add(subId);
                    continue;
                }
                cpage  = cpage.Substring(keypos);
                cpage  = cpage.Substring(cpage.IndexOf("/", StringComparison.Ordinal));
                sp.URL = "https:"
                         + cpage.Substring(0, cpage.IndexOf("\"", StringComparison.Ordinal));

                // processing submission description; also extracts submission date and title
                {
                    Utils.FillPropertiesFromDateTime(DateTime.Now, sp); // set Now as a fallback date
                    sp.TITLE = "Unknown";                               // fallback title

                    // title
                    const string key_title  = @"<div class=""submission-title"">";
                    const string key_enddiv = "</div>";
                    cpage = cpage.Substring(cpage.IndexOf(key_title, StringComparison.Ordinal));
                    string sub_title_div = cpage.Substring(0,
                                                           cpage.IndexOf(key_enddiv, cpage.IndexOf(key_enddiv, StringComparison.Ordinal) + 1,
                                                                         StringComparison.Ordinal) + key_enddiv.Length);
                    var titleMatch = Regex.Match(sub_title_div, "<h2><p>(.+?)</p></h2>", RegexOptions.CultureInvariant);
                    if (titleMatch.Success)
                    {
                        sp.TITLE = Utils.StripIllegalFilenameChars(titleMatch.Groups[1].Value);
                        Console.WriteLine("Title: " + sp.TITLE);
                    }
                    else
                    {
                        Console.WriteLine("Warning :: no submission title found!");
                    }

                    // replace relative date with the absolute one
                    string sub_date_strong = "";
                    var    dateMatch       = Regex.Match(cpage, "<strong.+?title=\"(.+?)\" class=\"popup_date\">(.+?)<.+?</strong>", RegexOptions.CultureInvariant);
                    if (dateMatch.Success)
                    {
                        string dateMatchVal     = dateMatch.Value;
                        string dateTimeStr      = dateMatch.Groups[1].Value;                    // fixed format date
                        string dateTimeStrFuzzy = dateMatch.Groups[2].Value;

                        // depending on user settings, fuzzy and fixed times may be swapped
                        if (dateTimeStrFuzzy.Contains(" PM") || dateTimeStrFuzzy.Contains(" AM"))
                        {
                            var temporary = dateTimeStr;
                            dateTimeStr      = dateTimeStrFuzzy;
                            dateTimeStrFuzzy = temporary;
                        }

                        // replace relative date with a fixed format one
                        sub_date_strong = dateMatchVal.Replace(dateTimeStrFuzzy, dateTimeStr);

                        // parse date
                        dateTimeStr = dateTimeStr.Replace(",", "");
                        {
                            const string dateFormat = "MMM d yyyy hh:mm tt";
                            try
                            {
                                DateTime dateTime = DateTime.ParseExact(dateTimeStr, dateFormat, CultureInfo.InvariantCulture);
                                Utils.FillPropertiesFromDateTime(dateTime, sp);
                            }
                            catch (Exception e)
                            {
                                Console.WriteLine("Warning :: cannot parse date :: " + e.Message);
                                Console.WriteLine("Info :: date string :: " + dateTimeStr);
                            }
                        }
                    }
                    else
                    {
                        Console.WriteLine("Warning :: unable to extact submission date");
                    }

                    // extract description
                    const string key_desc = @"<div class=""submission-description user-submitted-links"">";
                    cpage = cpage.Substring(cpage.IndexOf(key_desc, StringComparison.Ordinal));
                    cpage = cpage.Substring(0,
                                            cpage.IndexOf(key_enddiv, cpage.IndexOf(key_enddiv, StringComparison.Ordinal) + 1,
                                                          StringComparison.Ordinal) + key_enddiv.Length);
                    cpage = cpage.Replace("href=\"/", "href=\"https://furaffinity.net/");
                    cpage = cpage.Replace("src=\"//", "src=\"https://");

                    cpage = @"<div class=""submission-description-container link-override"">
                        <div class=""submission-title"">
                            <h2 class=""submission-title-header"">{{{title}}}</h2>
                            Posted {{{date}}}
                        </div><hr>".Replace("{{{title}}}", sp.TITLE).Replace("{{{date}}}", sub_date_strong) + cpage;
                }

                sp.ARTIST   = sp.URL.Substring(sp.URL.LastIndexOf(@"/art/") + 5);
                sp.ARTIST   = sp.ARTIST.Substring(0, sp.ARTIST.IndexOf('/'));
                sp.FILEFULL = sp.URL.Substring(sp.URL.LastIndexOf('/') + 1);
                sp.FILEFULL = Utils.StripIllegalFilenameChars(sp.FILEFULL);
                sp.FILEID   = sp.FILEFULL.Substring(0, sp.FILEFULL.IndexOf('.'));
                if (sp.FILEFULL.IndexOf('_') >= 0) // valid filename (some names on FA are corrupted and contain nothing but '.' after ID)
                {
                    sp.FILEPART = sp.FILEFULL.Substring(sp.FILEFULL.IndexOf('_') + 1);
                    if (sp.FILEFULL.LastIndexOf('.') >= 0) // has extension
                    {
                        sp.EXT = (sp.FILEFULL + " ").Substring(sp.FILEFULL.LastIndexOf('.') + 1).TrimEnd();
                        if (sp.EXT.CompareTo("") == 0)
                        {
                            sp.EXT = @"jpg";
                        }
                    }
                    else
                    {
                        sp.EXT = @"jpg";
                    }
                }
                else
                {
                    sp.FILEPART = @"unknown.jpg";
                    sp.EXT      = @"jpg";
                }

                // apply template(s)
                string fname  = GlobalSettings.Settings.filenameTemplate;
                string dfname = GlobalSettings.Settings.descrFilenameTemplate;
                foreach (FieldInfo fi
                         in sp.GetType().GetFields(
                             BindingFlags.Instance | BindingFlags.Public).ToArray()
                         )
                {
                    if (fi.FieldType == typeof(string))
                    {
                        fname  = fname.Replace("%" + fi.Name + "%", (string)fi.GetValue(sp));
                        dfname = dfname.Replace("%" + fi.Name + "%", (string)fi.GetValue(sp));
                    }
                }

                // make sure directories exist
                string fnamefull  = Path.Combine(GlobalSettings.Settings.downloadPath, fname);
                string dfnamefull = Path.Combine(GlobalSettings.Settings.downloadPath, dfname);
                try
                {
                    Directory.CreateDirectory(Path.GetDirectoryName(fnamefull));
                    Directory.CreateDirectory(Path.GetDirectoryName(dfnamefull));
                }
                catch
                {
                    Console.WriteLine("Failed to make sure target directories do exist.");
                    break;
                }

                // save description
                if (needDescription)
                {
                    try
                    {
                        File.WriteAllText(dfnamefull, cpage);
                        Console.WriteLine("description saved to filename:" + dfname);
                    }
                    catch (Exception E)
                    {
                        Console.WriteLine("Error saving description:" + E.Message);
                    }
                }

                // download file
                Console.WriteLine("target filename: " + fname);
                if (File.Exists(fnamefull))
                {
                    SubmissionsDB.DB.AddSubmission(uint.Parse(subId));
                    Console.WriteLine("Already exists, continuing~");
                    continue;
                }
                int fattempts = 3;
fbeforeawait:
                try
                {
                    Console.WriteLine("Downloading: " + sp.URL);
                    using (
                        Stream contentStream = await(
                            await http.GetAsync(sp.URL, HttpCompletionOption.ResponseHeadersRead)
                            ).Content.ReadAsStreamAsync(),
                        stream = new FileStream(
                            fnamefull,
                            FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 /*Mb*/, true
                            )
                        )
                    {
                        await ReadNetworkStream(contentStream, stream, 5000);

                        // await contentStream.CopyToAsync(stream); // this works, but may hang forever in case of network errors
                        SubmissionsDB.DB.AddSubmission(uint.Parse(subId));
                    }
                }
                catch (Exception E)
                {
                    // write error message
                    if (E is ObjectDisposedException)
                    {
                        Console.WriteLine("Network error (data receive timeout)");
                    }
                    else
                    {
                        Console.WriteLine("GET request error (file " + sp.FILEID + "): " + E.Message);
                    }
                    // remove incomplete download
                    if (File.Exists(fnamefull))
                    {
                        File.Delete(fnamefull);
                    }
                    // try again or abort operation
                    fattempts--;
                    System.Threading.Thread.Sleep(2000);
                    if (fattempts > 0)
                    {
                        goto fbeforeawait;
                    }
                    {
                        Console.WriteLine("Giving up on downloading {0}", subId);
                        res.failedToDownload.Add(subId);
                        continue;
                    }
                }
                Console.WriteLine("Done: #" + subId);
                TaskbarProgress.SetValue(currentConsoleHandle, subs.Count - i, subs.Count);
                res.processedPerfectly++;
            }

            // writing results
            try
            {
                if (res.failedToGetPage.Count > 0 || res.failedToDownload.Count > 0)
                {
                    File.WriteAllLines(Path.Combine(GlobalSettings.Settings.systemPath, "get_sub_page_failed.log"), res.failedToGetPage);
                    File.WriteAllLines(Path.Combine(GlobalSettings.Settings.systemPath, "download_failed.log"), res.failedToGetPage);
                }
            }
            catch (Exception E)
            {
                Console.WriteLine("Failed to save list of subs with issues: " + E.Message);
            }
            // save DB
            SubmissionsDB.Save();
            // stop progress indicating
            TaskbarProgress.SetState(currentConsoleHandle, TaskbarProgress.TaskbarStates.NoProgress);
            // return result, actually
            return(res);
        }
Пример #6
0
        public async Task <ProcessingResults> ProcessSubmissionsList(List <string> subs, bool needDescription)
        {
            Console.WriteLine("Processing submissions list...");
            ProcessingResults res = new ProcessingResults();

            // iterate over all the submissions in list
            for (int i = subs.Count - 1; i >= 0; i--)
            {
                string subId = subs[i];
                // don't care about empty strings
                if (subId == null || subId.CompareTo("") == 0)
                {
                    continue;
                }
                Console.WriteLine("> Processing submission #" + subId);
                // check if in DB already
                try
                {
                    if (SubmissionsDB.DB.Exists(uint.Parse(subId)) &&
                        GlobalSettings.Settings.downloadOnlyOnce)
                    {
                        Console.WriteLine("Skipped (present in DB)");
                        continue;
                    }
                }
                catch
                {
                    Console.WriteLine("Unexpected error (DB presence check failed)!");
                    continue;
                }
                string subUrl = "https://www.furaffinity.net/view/" + subId;
                // get submission page
                int    attempts = 3;
                string cpage    = "";
beforeawait:
                try
                {
                    Console.WriteLine("Getting page: " + subUrl);
                    cpage = await http.GetStringAsync(subUrl);
                }
                catch (Exception E)
                {
                    Console.WriteLine("GET request error (" + subUrl + "): " + E.Message);
                    attempts--;
                    System.Threading.Thread.Sleep(2000);
                    if (attempts > 0)
                    {
                        goto beforeawait;
                    }
                    else
                    {
                        Console.WriteLine("Giving up on #" + subId);
                        res.failedToGetPage.Add(subId);
                        continue;
                    }
                }
                // process submission page
                string          downbtnkey = "<a href=\"//d.facdn.net/";
                string          desckey    = "<div class=\"submission-description-container";
                SubmissionProps sp         = new SubmissionProps();
                sp.SUBMID = subId;
                int keypos = cpage.IndexOf(downbtnkey, StringComparison.Ordinal);
                if (keypos < 0)
                {
                    Console.WriteLine("[Warning] got page, but it doesn't contain any download links.");
                    res.failedToGetPage.Add(subId);
                    continue;
                }
                cpage  = cpage.Substring(keypos);
                cpage  = cpage.Substring(cpage.IndexOf("/", StringComparison.Ordinal));
                sp.URL = "https:"
                         + cpage.Substring(0, cpage.IndexOf("\"", StringComparison.Ordinal));
                if (needDescription)
                {
                    cpage = cpage.Substring(cpage.IndexOf(desckey, StringComparison.Ordinal));
                    string desckeyend = "</div>";
                    cpage = cpage.Substring(0,
                                            cpage.IndexOf(desckeyend, cpage.IndexOf(desckeyend) + 1) + desckeyend.Length
                                            );
                    cpage = cpage.Replace("href=\"/", "href=\"https://furaffinity.net/");
                    cpage = cpage.Replace("src=\"//", "src=\"https://");
                }
                sp.ARTIST   = sp.URL.Substring(sp.URL.LastIndexOf(@"/art/") + 5);
                sp.ARTIST   = sp.ARTIST.Substring(0, sp.ARTIST.IndexOf('/'));
                sp.FILEFULL = sp.URL.Substring(sp.URL.LastIndexOf('/') + 1);
                sp.FILEFULL = string.Concat(sp.FILEFULL.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries));
                sp.FILEID   = sp.FILEFULL.Substring(0, sp.FILEFULL.IndexOf('.'));
                if (sp.FILEFULL.IndexOf('_') >= 0) // valid filename (some names on FA are corrupted and contain nothing but '.' after ID)
                {
                    sp.FILEPART = sp.FILEFULL.Substring(sp.FILEFULL.IndexOf('_') + 1);
                    if (sp.FILEFULL.LastIndexOf('.') >= 0) // has extension
                    {
                        sp.EXT = (sp.FILEFULL + " ").Substring(sp.FILEFULL.LastIndexOf('.') + 1).TrimEnd();
                        if (sp.EXT.CompareTo("") == 0)
                        {
                            sp.EXT = @"jpg";
                        }
                    }
                    else
                    {
                        sp.EXT = @"jpg";
                    }
                }
                else
                {
                    sp.FILEPART = @"unknown.jpg";
                    sp.EXT      = @"jpg";
                }
                // apply template(s)
                string fname  = GlobalSettings.Settings.filenameTemplate;
                string dfname = GlobalSettings.Settings.descrFilenameTemplate;
                foreach (FieldInfo fi
                         in sp.GetType().GetFields(
                             BindingFlags.Instance | BindingFlags.Public).ToArray()
                         )
                {
                    if (fi.FieldType == typeof(string))
                    {
                        fname  = fname.Replace("%" + fi.Name + "%", (string)fi.GetValue(sp));
                        dfname = dfname.Replace("%" + fi.Name + "%", (string)fi.GetValue(sp));
                    }
                }
                // make sure directories exist
                string fnamefull  = Path.Combine(GlobalSettings.Settings.downloadPath, fname);
                string dfnamefull = Path.Combine(GlobalSettings.Settings.downloadPath, dfname);
                try
                {
                    Directory.CreateDirectory(Path.GetDirectoryName(fnamefull));
                    Directory.CreateDirectory(Path.GetDirectoryName(dfnamefull));
                }
                catch
                {
                    Console.WriteLine("Failed to make sure target directories do exist.");
                    break;
                }
                // save description
                if (needDescription)
                {
                    try
                    {
                        File.WriteAllText(dfnamefull, cpage);
                        Console.WriteLine("description saved to filename:" + dfname);
                    }
                    catch (Exception E)
                    {
                        Console.WriteLine("Error saving description:" + E.Message);
                    }
                }
                // download file
                Console.WriteLine("target filename: " + fname);
                if (File.Exists(fnamefull))
                {
                    SubmissionsDB.DB.AddSubmission(uint.Parse(subId));
                    Console.WriteLine("Already exists, continuing~");
                    continue;
                }
                int fattempts = 3;
fbeforeawait:
                try
                {
                    Console.WriteLine("Downloading: " + sp.URL);
                    using (
                        Stream contentStream = await(
                            await http.GetAsync(sp.URL, HttpCompletionOption.ResponseHeadersRead)
                            ).Content.ReadAsStreamAsync(),
                        stream = new FileStream(
                            fnamefull,
                            FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 /*Mb*/, true
                            )
                        )
                    {
                        await ReadNetworkStream(contentStream, stream, 5000);

                        // await contentStream.CopyToAsync(stream); // this works, but may hang forever in case of network errors
                        SubmissionsDB.DB.AddSubmission(uint.Parse(subId));
                    }
                }
                catch (Exception E)
                {
                    // write error message
                    if (E is ObjectDisposedException)
                    {
                        Console.WriteLine("Network error (data receive timeout)");
                    }
                    else
                    {
                        Console.WriteLine("GET request error (file " + sp.FILEID + "): " + E.Message);
                    }
                    // remove incomplete download
                    if (File.Exists(fnamefull))
                    {
                        File.Delete(fnamefull);
                    }
                    // try again or abort operation
                    fattempts--;
                    System.Threading.Thread.Sleep(2000);
                    if (fattempts > 0)
                    {
                        goto fbeforeawait;
                    }
                    {
                        Console.WriteLine("Giving up on downloading {0}", subId);
                        res.failedToDownload.Add(subId);
                        continue;
                    }
                }
                Console.WriteLine("Done: #" + subId);
                TaskbarProgress.SetValue(currentConsoleHandle, subs.Count - i, subs.Count);
                res.processedPerfectly++;
            }
            // writing results
            try
            {
                if (res.failedToGetPage.Count > 0 || res.failedToDownload.Count > 0)
                {
                    File.WriteAllLines(Path.Combine(GlobalSettings.Settings.systemPath, "get_sub_page_failed.log"), res.failedToGetPage);
                    File.WriteAllLines(Path.Combine(GlobalSettings.Settings.systemPath, "download_failed.log"), res.failedToGetPage);
                }
            }
            catch (Exception E)
            {
                Console.WriteLine("Failed to save list of subs with issues: " + E.Message);
            }
            // save DB
            SubmissionsDB.Save();
            // stop progress indicating
            TaskbarProgress.SetState(currentConsoleHandle, TaskbarProgress.TaskbarStates.NoProgress);
            // return result, actually
            return(res);
        }
Пример #7
0
        public async Task <ProcessingResults> ProcessSubmissionsList(List <string> subs, bool needDescription, bool updateMode = false)
        {
            Console.WriteLine("Processing submissions list...");
            ProcessingResults res = new ProcessingResults();

            // iterate over all the submissions in list
            for (int i = subs.Count - 1; i >= 0; i--)
            {
                // expected format: ID#FileID@attributes
                // everything except for ID is optional
                string subStr = subs[i];
                if (string.IsNullOrEmpty(subStr))
                {
                    continue;                               // don't care about empty strings
                }
                string subId;
                uint   subIdInt   = 0;
                uint   subFid     = 0;
                uint   subInitFid = 0;
                bool   aScraps    = false;

                const string subIdRegex = @"^(?<id>[0-9]+?)(#(?<fid>[0-9]+?)){0,1}(@(?<attr>.+?)){0,1}$";
                var          subIdMatch = Regex.Match(subStr, subIdRegex);
                if (subIdMatch.Success)
                {
                    subId = subIdMatch.Groups["id"].Value;
                    uint.TryParse(subId, out subIdInt);
                    if (subIdMatch.Groups["fid"].Success)
                    {
                        uint.TryParse(subIdMatch.Groups["fid"].Value, out subFid);
                    }
                    /// Attributes section has only been used for (terrible) scraps detection;
                    /// a better method is now implemented, making the section useless
                    //if (subIdMatch.Groups["attr"].Success)
                    //{
                    //    string attributes = subIdMatch.Groups["attr"].Value;
                    //    if (attributes.Contains("s")) aScraps = true;
                    //}
                }
                else
                {
                    Console.WriteLine("Error :: Malformed submission ID: " + subStr);
                    continue;
                }

                uint dbSubFid    = SubmissionsDB.DB.GetFileId(subIdInt);
                bool dbSubExists = SubmissionsDB.DB.Exists(subIdInt);

                Console.WriteLine(string.Format("> Processing submission {0} {1}",
                                                subId,
                                                subFid > 0 ? string.Format("(file id {0})", subFid) : ""
                                                ));

                // Skip submissions that can be skipped without making any network requests
                try
                {
                    if (dbSubExists && GlobalSettings.Settings.downloadOnlyOnce)
                    {
                        // can skip at lowest cost if either:
                        // * not in update mode
                        // * file ID is known and matches the one stored in the DB
                        if ((!updateMode) || (updateMode && dbSubFid == subFid && dbSubFid != 0))
                        {
                            Console.WriteLine("Skipped (present in DB)");
                            continue;
                        }
                        else
                        {
                            Console.WriteLine("Submission is present in the DB, but may have been updated; re-checking~");
                        }
                    }
                }
                catch
                {
                    Console.WriteLine("Unexpected error (DB presence check failed)!");
                    continue;
                }

                // get submission page
                string subUrl   = "https://www.furaffinity.net/view/" + subId;
                int    attempts = 3;
                string cpage    = "";
beforeawait:
                try
                {
                    Console.WriteLine("Getting page: " + subUrl);
                    cpage = await http.GetStringAsync(subUrl);
                }
                catch (Exception E)
                {
                    Console.WriteLine("GET request error (" + subUrl + "): " + E.Message);
                    attempts--;
                    System.Threading.Thread.Sleep(2000);
                    if (attempts > 0)
                    {
                        goto beforeawait;
                    }
                    else
                    {
                        Console.WriteLine("Giving up on #" + subId);
                        res.failedToGetPage.Add(subId);
                        continue;
                    }
                }

                // process submission page
                var downbtnkeys = new string[] {
                    "<a href=\"//d.facdn.net/", "<a href=\"//d2.facdn.net/",
                    "<a href=\"//d.furaffinity.net/"
                };
                SubmissionProps sp = new SubmissionProps();
                sp.SUBMID = subId;
                int keypos = -1;
                foreach (var downbtnkey in downbtnkeys)
                {
                    keypos = cpage.IndexOf(downbtnkey, StringComparison.Ordinal);
                    if (keypos >= 0)
                    {
                        break;
                    }
                }
                if (keypos < 0)
                {
                    Console.WriteLine("[Error] got page, but it doesn't contain any download links.");
                    res.failedToGetPage.Add(subId);
                    continue;
                }
                cpage  = cpage.Substring(keypos);
                cpage  = cpage.Substring(cpage.IndexOf("/", StringComparison.Ordinal));
                sp.URL = "https:"
                         + cpage.Substring(0, cpage.IndexOf("\"", StringComparison.Ordinal));

                #region download URL parsing
                bool         extensionInvalid   = false; // future use, possibly come up with an extension that makes sense on a case by case basis
                const string urlComponentsRegex = @"\/art\/(?<artist>.+?)\/.*?(?<curfid>\d+?)\/(?<fid>.+?)\.(?<fname>.*)$";
                var          urlCompMatch       = Regex.Match(sp.URL, urlComponentsRegex);
                if (urlCompMatch.Success)
                {
                    sp.ARTIST    = urlCompMatch.Groups["artist"].Value;
                    sp.CURFILEID = urlCompMatch.Groups["curfid"].Value;
                    uint.TryParse(sp.CURFILEID, out subFid);
                    sp.FILEID = urlCompMatch.Groups["fid"].Value;
                    uint.TryParse(sp.FILEID, out subInitFid);
                    string filename = urlCompMatch.Groups["fname"].Value;
                    /// original filename usually follows this pattern:
                    ///     $file_id.$artist_originalFileName.ext
                    ///             [^ "fname" group value       ]
                    /// however, some old (~2006) submissions use this pattern instead:
                    ///     $file_id.$artist.originalFileName.ext
                    /// it is also quite common for the fname to be blank, i.e.
                    ///     $file_id.
                    /// in this case we have no choice but to come up with our own name
                    var fnameCheckMatch = Regex.Match(filename, string.Format(@"^{0}[_.](.+)", Regex.Escape(sp.ARTIST)));
                    if (fnameCheckMatch.Success)
                    {
                        var filepart = fnameCheckMatch.Groups[1].Value;
                        if (filepart.EndsWith(".") || !filepart.Contains(".")) // no extension or an empty one
                        {
                            extensionInvalid = true;
                            Console.WriteLine("Info :: missing filename extension, assuming .jpg");
                            if (filepart.EndsWith("."))
                            {
                                filepart = filepart.Substring(0, filepart.Length - 1) + ".jpg";
                            }
                            else
                            {
                                filepart = filepart + ".jpg";
                            }
                        }
                        var filepartDotSplit = filepart.Split(new char[] { '.' });
                        sp.FILEPART   = filepart;
                        sp.FILEPARTNE = string.Join(".", filepartDotSplit.Take(filepartDotSplit.Length - 1));
                        sp.EXT        = Utils.StripIllegalFilenameChars(filepartDotSplit.Last());
                    }
                    else // completely broken filenames get replaced with "unknown.jpg"
                    {
                        Console.WriteLine("Info :: broken filename detected, replacing with \"unknown.jpg\"");
                        sp.FILEPART      = "unknown.jpg";
                        sp.FILEPARTNE    = "unknown";
                        sp.EXT           = "jpg";
                        extensionInvalid = true;
                    }

                    sp.FILEFULL = sp.FILEID + "." + sp.ARTIST + "_" + sp.FILEPART;

                    sp.FILEFULL = Utils.StripIllegalFilenameChars(sp.FILEFULL);
                    sp.FILEPART = Utils.StripIllegalFilenameChars(sp.FILEPART);
                }
                else
                {
                    Console.WriteLine("Error: could not make sense of the URL for submission " + subId);
                    res.failedToDownload.Add(subId);
                    continue;
                }
                #endregion

                #region scraps detection, submission date, title and description
                {
                    const string key_title    = @"<div class=""submission-title"">";
                    const string key_enddiv   = "</div>";
                    var          submTitlePos = cpage.IndexOf(key_title, StringComparison.Ordinal);

                    // scraps check: if there is a link to /$user/scraps before the submission title, it's in scraps
                    var scrapsCheckMatch = Regex.Match(cpage, string.Format(@"href=""/scraps/{0}/""", Regex.Escape(sp.ARTIST)));
                    if (scrapsCheckMatch.Success && scrapsCheckMatch.Index < submTitlePos)
                    {
                        Console.WriteLine("Location: scraps");
                        aScraps = true;
                    }
                    else
                    {
                        Console.WriteLine("Location: main gallery");
                        aScraps = false;
                    }

                    Utils.FillPropertiesFromDateTime(DateTime.Now, sp); // set Now as a fallback date
                    sp.TITLE = "Unknown";                               // fallback title

                    // title
                    cpage = cpage.Substring(submTitlePos);
                    string sub_title_div = cpage.Substring(0,
                                                           cpage.IndexOf(key_enddiv, cpage.IndexOf(key_enddiv, StringComparison.Ordinal) + 1,
                                                                         StringComparison.Ordinal) + key_enddiv.Length);
                    var titleMatch = Regex.Match(sub_title_div, "<h2><p>(.+?)</p></h2>", RegexOptions.CultureInvariant);
                    if (titleMatch.Success)
                    {
                        sp.TITLE = Utils.StripIllegalFilenameChars(System.Net.WebUtility.HtmlDecode(titleMatch.Groups[1].Value));
                        Console.WriteLine("Title: " + sp.TITLE);
                    }
                    else
                    {
                        Console.WriteLine("Warning :: no submission title found!");
                    }

                    // replace relative date with the absolute one
                    string sub_date_strong = "";
                    var    dateMatch       = Regex.Match(cpage, "<strong.+?title=\"(.+?)\" class=\"popup_date\">(.+?)<.+?</strong>", RegexOptions.CultureInvariant);
                    if (dateMatch.Success)
                    {
                        string dateMatchVal     = dateMatch.Value;
                        string dateTimeStr      = dateMatch.Groups[1].Value; // fixed format date
                        string dateTimeStrFuzzy = dateMatch.Groups[2].Value;

                        // depending on user settings, fuzzy and fixed times may be swapped
                        if (dateTimeStrFuzzy.Contains(" PM") || dateTimeStrFuzzy.Contains(" AM"))
                        {
                            var temporary = dateTimeStr;
                            dateTimeStr      = dateTimeStrFuzzy;
                            dateTimeStrFuzzy = temporary;
                        }

                        // replace relative date with a fixed format one
                        sub_date_strong = dateMatchVal.Replace(dateTimeStrFuzzy, dateTimeStr);

                        // parse date
                        dateTimeStr = dateTimeStr.Replace(",", "");
                        {
                            const string dateFormat = "MMM d yyyy hh:mm tt";
                            try
                            {
                                DateTime dateTime = DateTime.ParseExact(dateTimeStr, dateFormat, CultureInfo.InvariantCulture);
                                Utils.FillPropertiesFromDateTime(dateTime, sp);
                            }
                            catch (Exception e)
                            {
                                Console.WriteLine("Warning :: cannot parse date :: " + e.Message);
                                Console.WriteLine("Info :: date string :: " + dateTimeStr);
                            }
                        }
                    }
                    else
                    {
                        Console.WriteLine("Warning :: unable to extact submission date");
                    }

                    // extract description
                    const string key_desc = @"<div class=""submission-description user-submitted-links"">";
                    cpage = cpage.Substring(cpage.IndexOf(key_desc, StringComparison.Ordinal));
                    cpage = cpage.Substring(0,
                                            cpage.IndexOf(key_enddiv, cpage.IndexOf(key_enddiv, StringComparison.Ordinal) + 1,
                                                          StringComparison.Ordinal) + key_enddiv.Length);
                    cpage = cpage.Replace("href=\"/", "href=\"https://furaffinity.net/");
                    cpage = cpage.Replace("src=\"//", "src=\"https://");

                    cpage = @"<div class=""submission-description-container link-override"">
                        <div class=""submission-title"">
                            <h2 class=""submission-title-header"">{{{title}}}</h2>
                            Posted {{{date}}}
                        </div><hr>".Replace("{{{title}}}", sp.TITLE).Replace("{{{date}}}", sub_date_strong) + cpage;
                }
                #endregion

                // apply template(s)
                string fname          = GlobalSettings.Settings.filenameTemplate;
                string dfname         = GlobalSettings.Settings.descrFilenameTemplate;
                var    scrapsTemplate = aScraps ? GlobalSettings.Settings.scrapsTemplateActive : GlobalSettings.Settings.scrapsTemplatePassive;
                fname  = fname.Replace("%SCRAPS%", scrapsTemplate);
                dfname = dfname.Replace("%SCRAPS%", scrapsTemplate);
                foreach (FieldInfo fi
                         in sp.GetType().GetFields(
                             BindingFlags.Instance | BindingFlags.Public).ToArray()
                         )
                {
                    if (fi.FieldType == typeof(string))
                    {
                        fname  = fname.Replace("%" + fi.Name + "%", (string)fi.GetValue(sp));
                        dfname = dfname.Replace("%" + fi.Name + "%", (string)fi.GetValue(sp));
#if DEBUG_PRINT_ALL_TEMPLATE_VALS
                        // debug only: output all template values:
                        Console.WriteLine(string.Format("+++ {0} = {1}", "%" + fi.Name + "%", (string)fi.GetValue(sp)));
#endif
                    }
                }

                // make sure directories exist
                string fnamefull  = Path.Combine(GlobalSettings.Settings.downloadPath, fname);
                string dfnamefull = Path.Combine(GlobalSettings.Settings.downloadPath, dfname);
                try
                {
                    Directory.CreateDirectory(Path.GetDirectoryName(fnamefull));
                    Directory.CreateDirectory(Path.GetDirectoryName(dfnamefull));
                }
                catch
                {
                    Console.WriteLine("Failed to make sure target directories do exist.");
                    break;
                }

                // save description
                if (needDescription)
                {
                    try
                    {
                        File.WriteAllText(dfnamefull, cpage);
                        Console.WriteLine("description saved to filename:" + dfname);
                    }
                    catch (Exception E)
                    {
                        Console.WriteLine("Error saving description:" + E.Message);
                    }
                }

                var fileExists = File.Exists(fnamefull);

                Console.WriteLine("target filename: " + fname + (fileExists ? " (exists)" : ""));

                // at this point we have the actual file ID, and can skip downloading based on that
                if (GlobalSettings.Settings.downloadOnlyOnce)
                {
                    if ((!updateMode) && fileExists) // checked earlier: && !dbSubExists
                    {
                        if (subInitFid != subFid)
                        {
                            Console.WriteLine(string.Format(
                                                  "Note :: submission {0} exists locally, but could've been updated\n"
                                                  + "consider running this task in update mode", subId
                                                  ));
                            SubmissionsDB.DB.AddSubmission(subIdInt);
                        }
                        else
                        {
                            Console.WriteLine("Already exists, continuing~");
                            SubmissionsDB.DB.AddSubmissionWithFileId(subIdInt, subFid);
                        }
                        continue;
                    }
                    // this exact check can also be found before, it is repeated here for cases
                    // when subFid was not known before the submission page request
                    if (updateMode && dbSubFid == subFid && dbSubFid != 0)
                    {
                        Console.WriteLine("Already downloaded, continuing~");
                        continue;
                    }
                }
                else // not `download only once`
                {
                    if ((!updateMode) && fileExists)
                    {
                        if (subInitFid != subFid)
                        {
                            SubmissionsDB.DB.AddSubmission(subIdInt);
                        }
                        else
                        {
                            SubmissionsDB.DB.AddSubmissionWithFileId(subIdInt, subFid);
                        }
                        Console.WriteLine("Already exists, continuing~");
                        continue;
                    }
                    if (updateMode && fileExists && dbSubFid == subFid && dbSubFid != 0)
                    {
                        Console.WriteLine("Already exists, continuing~");
                        continue;
                    }
                }

                // if we got here, there was no reason to skip the download
                bool   mayBeUselessDownload = false;
                string oldFileHash          = "";
                if (fileExists)
                {
                    Console.WriteLine(string.Format("subfid {0}   dbsf {1}", subFid, dbSubFid));
                    oldFileHash = Utils.FileHash(fnamefull);
                    fnamefull   = Path.Combine(GlobalSettings.Settings.downloadPath,
                                               string.Format("{1} [v.{0}].{2}", subFid, fname, sp.EXT));
                    if (!(subFid != dbSubFid && dbSubFid != 0))
                    {
                        Console.WriteLine("Info :: stored metadata is insufficient; downloading a remote file to compare aganst local");
                        mayBeUselessDownload = true;
                    }
                }


                // download file
                int fattempts = 3;
fbeforeawait:
                try
                {
                    Console.WriteLine("Downloading: " + sp.URL);
                    // "?" can only be in the URL if the user named their submission this way
                    // it WILL be mistreated as an URL parameter, but this dirty hack with explicit replacement fixes it
                    using (var response = await http.GetAsync(sp.URL.Replace("?", "%3F"), HttpCompletionOption.ResponseHeadersRead))
                    {
                        if (!response.IsSuccessStatusCode)
                        {
                            throw new Exception(string.Format("HTTP error: {0}", response.StatusCode));
                        }
                        using (
                            Stream contentStream = await response.Content.ReadAsStreamAsync(),
                            stream = new FileStream(
                                fnamefull,
                                FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 1024 /*Mb*/, true
                                )
                            )
                        {
                            await ReadNetworkStream(contentStream, stream, 5000);

                            // await contentStream.CopyToAsync(stream); // this works, but may hang forever in case of network errors
                        }
                    }
                }
                catch (Exception E)
                {
                    // write error message
                    if (E is ObjectDisposedException)
                    {
                        Console.WriteLine("Network error (data receive timeout)");
                    }
                    else
                    {
                        Console.WriteLine("GET request error (file " + sp.FILEID + "): " + E.Message);
                    }
                    // remove incomplete download
                    if (File.Exists(fnamefull))
                    {
                        File.Delete(fnamefull);
                    }
                    // try again or abort operation
                    fattempts--;
                    System.Threading.Thread.Sleep(2000);
                    if (fattempts > 0)
                    {
                        goto fbeforeawait;
                    }
                    {
                        Console.WriteLine("Giving up on downloading {0}", subId);
                        res.failedToDownload.Add(subId);
                        continue;
                    }
                }

                SubmissionsDB.DB.AddSubmissionWithFileId(subIdInt, subFid);
                if (mayBeUselessDownload)
                {
                    var newFileHash = Utils.FileHash(fnamefull);
                    if (newFileHash == oldFileHash)
                    {
                        Console.WriteLine("Note :: existing version matches the one on the server, removing a duplicate");
                        File.Delete(fnamefull);
                    }
                }

                Console.WriteLine("Done: #" + subId);
                TaskbarProgress.SetValue(currentConsoleHandle, subs.Count - i, subs.Count);
                res.processedPerfectly++;
            }

            // writing results
            try
            {
                if (res.failedToGetPage.Count > 0 || res.failedToDownload.Count > 0)
                {
                    File.WriteAllLines(Path.Combine(GlobalSettings.Settings.systemPath, "get_sub_page_failed.log"), res.failedToGetPage);
                    File.WriteAllLines(Path.Combine(GlobalSettings.Settings.systemPath, "download_failed.log"), res.failedToDownload);
                }
            }
            catch (Exception E)
            {
                Console.WriteLine("Failed to save list of subs with issues: " + E.Message);
            }

            // save DB
            SubmissionsDB.Save();
            // stop progress indicating
            TaskbarProgress.SetState(currentConsoleHandle, TaskbarProgress.TaskbarStates.NoProgress);
            // return result, actually
            return(res);
        }