private static async Task <PayPalAccessToken> GetPayPalAccessTokenAsync(IMyConfiguration configuration, HttpClient http)
{
string payPalClientId = configuration.GetValue("PayPalClientId");
string payPalClientSecret = configuration.GetValue("PayPalClientSecret");
byte[] bytes = Encoding.GetEncoding("iso-8859-1").GetBytes($"{payPalClientId}:{payPalClientSecret}");
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, "/v1/oauth2/token");
request.Headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(bytes));
var form = new Dictionary <string, string>
{
["grant_type"] = "client_credentials"
};
request.Content = new FormUrlEncodedContent(form);
HttpResponseMessage response = await http.SendAsync(request);
string content = await response.Content.ReadAsStringAsync();
var accessToken = JsonConvert.DeserializeObject <PayPalAccessToken>(content);
return(accessToken);
}
public async Task <List <T> > GetLatestAsync(IMyConfiguration configuration, TextWriter writer, bool forceUpdate = false)
{
var date = new Date();
var currentDate = date.CurrentDate;
var firstDayOfTheYear = date.FirstDayOfTheYear;
// default is a lookup from beginning of year until now
DateTime from = firstDayOfTheYear;
DateTime to = currentDate;
string cacheDir = configuration.GetValue("CacheDir");
if (forceUpdate)
{
await writer.WriteLineAsync(string.Format("Forcing updating from {0:yyyy-MM-dd} to {1:yyyy-MM-dd}", ForcedUpdateFromDate, to));
var values = await GetListAsync(configuration, writer, ForcedUpdateFromDate, to);
string forcedCacheFilePath = Path.Combine(cacheDir, string.Format("{0}-{1:yyyy-MM-dd}-{2:yyyy-MM-dd}.csv", CacheFileNamePrefix, ForcedUpdateFromDate, to));
Utils.WriteCacheFile(forcedCacheFilePath, values);
await writer.WriteLineAsync(string.Format("Successfully wrote file to {0}", forcedCacheFilePath));
return(values);
}
// check if we have a cache file
var lastCacheFileInfo = Utils.FindLastCacheFile(cacheDir, CacheFileNamePrefix);
// if the cache file object has values
if (lastCacheFileInfo != null && !lastCacheFileInfo.Equals(default(FileDate)))
{
// find values starting from when the cache file ends and until now
from = lastCacheFileInfo.To;
to = currentDate;
// if the from date is today, then we already have an updated file so use cache
if (lastCacheFileInfo.To.Date.Equals(currentDate.Date))
{
// use latest cache file (or update if the cache file is empty)
return(await GetListAsync(configuration, writer, lastCacheFileInfo.FilePath, from, to));
}
else if (from != firstDayOfTheYear)
{
// combine new and old values
var updatedValues = await GetCombinedUpdatedAndExistingAsync(configuration, writer, lastCacheFileInfo, from, to);
// and store to new file
string newCacheFilePath = Path.Combine(cacheDir, string.Format("{0}-{1:yyyy-MM-dd}-{2:yyyy-MM-dd}.csv", CacheFileNamePrefix, firstDayOfTheYear, to));
Utils.WriteCacheFile(newCacheFilePath, updatedValues);
await writer.WriteLineAsync(string.Format("Successfully wrote file to {0}", newCacheFilePath));
return(updatedValues);
}
}
// get updated transactions (or from cache file)
string cacheFilePath = Path.Combine(cacheDir, string.Format("{0}-{1:yyyy-MM-dd}-{2:yyyy-MM-dd}.csv", CacheFileNamePrefix, from, to));
return(await GetListAsync(configuration, writer, cacheFilePath, from, to));
}
public static SkandiabankenBankStatement GetLatestBankStatement(IMyConfiguration configuration, bool forceUpdate = false)
{
string cacheDir = configuration.GetValue("CacheDir");
string cacheFileNamePrefix = configuration.GetValue("SBankenAccountNumber");
string dateFromToRegexPattern = @"(\d{4}_\d{2}_\d{2})\-(\d{4}_\d{2}_\d{2})\.xlsx$";
var lastCacheFileInfo = Utils.FindLastCacheFile(cacheDir, cacheFileNamePrefix, dateFromToRegexPattern, "yyyy_MM_dd", "_");
var date = new Date();
var currentDate = date.CurrentDate;
var firstDayOfTheYear = date.FirstDayOfTheYear;
var yesterday = date.Yesterday;
// check if we have a cache file
DateTime from = default(DateTime);
DateTime to = default(DateTime);
// if the cache file object has values
if (!lastCacheFileInfo.Equals(default(KeyValuePair <DateTime, string>)))
{
from = lastCacheFileInfo.To;
to = yesterday;
// if the from date is today, then we already have an updated file so use cache
if (from.Date.Equals(to.Date))
{
// use latest cache file
return(ReadBankStatement(lastCacheFileInfo.FilePath));
}
}
// Download all from beginning of year until now
from = firstDayOfTheYear;
to = yesterday;
// check special case if yesterday is last year
if (from.Year != to.Year)
{
from = from.AddYears(-1);
}
// get updated bank statement
string bankStatementFilePath = DownloadBankStatement(configuration, from, to);
return(ReadBankStatement(bankStatementFilePath));
}
private static string GetAccessToken(IMyConfiguration configuration)
{
string payPalClientId = configuration.GetValue("PayPalClientId");
string payPalClientSecret = configuration.GetValue("PayPalClientSecret");
var config = new Dictionary <string, string>();
config.Add("mode", global::PayPal.Api.BaseConstants.LiveMode);
config.Add("clientId", payPalClientId);
config.Add("clientSecret", payPalClientSecret);
config[global::PayPal.Api.BaseConstants.HttpConnectionTimeoutConfig] = "30000";
config[global::PayPal.Api.BaseConstants.HttpConnectionRetryConfig] = "3";
string accessToken = new global::PayPal.Api.OAuthTokenCredential(config).GetAccessToken();
//var apiContext = new PayPal.Api.APIContext(accessToken) { Config = config };
//var payments = PayPal.Api.Payment.List(apiContext, null, "", null, "", "", DateTime.Now.AddDays(-25).ToString(), DateTime.Now.ToString());
return(accessToken);
}
public static async Task <List <StripeTransaction> > GetStripeChargeTransactionsAsync(IMyConfiguration configuration, DateTime from, DateTime to)
{
// get stripe configuration parameters
string stripeApiKey = configuration.GetValue("StripeApiKey");
StripeConfiguration.SetApiKey(stripeApiKey);
var chargeService = new StripeChargeService();
chargeService.ExpandBalanceTransaction = true;
chargeService.ExpandCustomer = true;
chargeService.ExpandInvoice = true;
var allCharges = new List <StripeCharge>();
var lastId = String.Empty;
int MAX_PAGINATION = 100;
int itemsExpected = MAX_PAGINATION;
while (itemsExpected == MAX_PAGINATION)
{
IEnumerable <StripeCharge> charges = null;
if (String.IsNullOrEmpty(lastId))
{
charges = await chargeService.ListAsync(
new StripeChargeListOptions()
{
Limit = MAX_PAGINATION,
Created = new StripeDateFilter
{
GreaterThanOrEqual = from,
LessThanOrEqual = to
}
});
itemsExpected = charges.Count();
}
else
{
charges = await chargeService.ListAsync(
new StripeChargeListOptions()
{
Limit = MAX_PAGINATION,
StartingAfter = lastId,
Created = new StripeDateFilter
{
GreaterThanOrEqual = from,
LessThanOrEqual = to
}
});
itemsExpected = charges.Count();
}
allCharges.AddRange(charges);
if (allCharges.Count() > 0)
{
lastId = charges.LastOrDefault().Id;
}
}
var stripeTransactions = new List <StripeTransaction>();
foreach (var charge in allCharges)
{
// only process the charges that were successfull, aka Paid
if (charge.Paid)
{
var stripeTransaction = new StripeTransaction();
stripeTransaction.TransactionID = charge.Id;
stripeTransaction.Created = charge.Created;
stripeTransaction.Paid = charge.Paid;
stripeTransaction.CustomerEmail = charge.Metadata["email"];
stripeTransaction.OrderID = charge.Metadata["order_id"];
stripeTransaction.Amount = (decimal)charge.Amount / 100;
stripeTransaction.Net = (decimal)charge.BalanceTransaction.Net / 100;
stripeTransaction.Fee = (decimal)charge.BalanceTransaction.Fee / 100;
stripeTransaction.Currency = charge.Currency;
stripeTransaction.Description = charge.Description;
stripeTransaction.Status = charge.Status;
decimal amountRefunded = (decimal)charge.AmountRefunded / 100;
if (amountRefunded > 0)
{
// if anything has been refunded
// don't add if amount refunded and amount is the same (full refund)
if (stripeTransaction.Amount - amountRefunded == 0)
{
continue;
}
else
{
stripeTransaction.Amount = stripeTransaction.Amount - amountRefunded;
stripeTransaction.Net = stripeTransaction.Net - amountRefunded;
}
}
stripeTransactions.Add(stripeTransaction);
}
}
return(stripeTransactions);
}
public static async Task <List <StripeTransaction> > GetStripePayoutTransactionsAsync(IMyConfiguration configuration, DateTime from, DateTime to)
{
// get stripe configuration parameters
string stripeApiKey = configuration.GetValue("StripeApiKey");
StripeConfiguration.SetApiKey(stripeApiKey);
var payoutService = new StripePayoutService();
var allPayoutTransactions = new List <StripePayout>();
var lastId = String.Empty;
int MAX_PAGINATION = 100;
int itemsExpected = MAX_PAGINATION;
while (itemsExpected == MAX_PAGINATION)
{
IEnumerable <StripePayout> charges = null;
if (String.IsNullOrEmpty(lastId))
{
charges = await payoutService.ListAsync(
new StripePayoutListOptions()
{
Limit = MAX_PAGINATION,
Created = new StripeDateFilter
{
GreaterThanOrEqual = from,
LessThanOrEqual = to
}
});
itemsExpected = charges.Count();
}
else
{
charges = await payoutService.ListAsync(
new StripePayoutListOptions()
{
Limit = MAX_PAGINATION,
StartingAfter = lastId,
Created = new StripeDateFilter
{
GreaterThanOrEqual = from,
LessThanOrEqual = to
}
});
itemsExpected = charges.Count();
}
allPayoutTransactions.AddRange(charges);
if (allPayoutTransactions.Count() > 0)
{
lastId = charges.LastOrDefault().Id;
}
}
var stripeTransactions = new List <StripeTransaction>();
foreach (var payoutTransaction in allPayoutTransactions)
{
// only process the charges that are payouts
if (payoutTransaction.Object == "payout")
{
var stripeTransaction = new StripeTransaction();
stripeTransaction.TransactionID = payoutTransaction.Id;
stripeTransaction.Created = payoutTransaction.Created;
stripeTransaction.AvailableOn = payoutTransaction.ArrivalDate;
stripeTransaction.Paid = (payoutTransaction.Status == "paid");
stripeTransaction.Amount = (decimal)payoutTransaction.Amount / 100;
stripeTransaction.Currency = payoutTransaction.Currency;
stripeTransaction.Description = payoutTransaction.Description;
stripeTransaction.Status = payoutTransaction.Status;
stripeTransactions.Add(stripeTransaction);
}
}
return(stripeTransactions);
}
public static string DownloadBankStatement(IMyConfiguration configuration, DateTime from, DateTime to)
{
string cacheDir = configuration.GetValue("CacheDir");
string userDataDir = configuration.GetValue("UserDataDir");
string sbankenMobilePhone = configuration.GetValue("SBankenMobilePhone");
string sbankenBirthDate = configuration.GetValue("SBankenBirthDate");
string cacheFileNamePrefix = configuration.GetValue("SBankenAccountNumber");
string sbankenAccountId = configuration.GetValue("SBankenAccountId");
string downloadFolderPath = Environment.GetEnvironmentVariable("USERPROFILE") + @"\Downloads\";
string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath");
var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath);
driver.Navigate().GoToUrl("https://secure.sbanken.no/Authentication/BankIdMobile");
var waitLoginPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitLoginPage.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
// login if login form is present
if (SeleniumUtils.IsElementPresent(driver, By.Id("MobilePhone")))
{
// https://secure.sbanken.no/Authentication/BankIDMobile
// input id MobilePhone - Mobilnummer (8 siffer)
// input it BirthDate - Fødselsdato (ddmmåå)
// submit value = Neste
// login if login form is present
if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='MobilePhone']")) &&
SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='BirthDate']")))
{
IWebElement mobilePhone = driver.FindElement(By.XPath("//input[@id='MobilePhone']"));
IWebElement birthDate = driver.FindElement(By.XPath("//input[@id='BirthDate']"));
mobilePhone.Clear();
mobilePhone.SendKeys(sbankenMobilePhone);
birthDate.Clear();
birthDate.SendKeys(sbankenBirthDate);
// use birth date field to submit form
birthDate.Submit();
var waitLoginIFrame = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitLoginIFrame.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
}
}
try
{
var waitMainPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitMainPage.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.UrlToBe("https://secure.sbanken.no/Home/Overview/Full#/"));
}
catch (WebDriverTimeoutException)
{
Console.WriteLine("Timeout - Logged in to Skandiabanken too late. Stopping.");
return(null);
}
// download account statement
string accountStatementDownload = string.Format("https://secure.sbanken.no/Home/AccountStatement/ViewExcel?AccountId={0}&CustomFromDate={1:dd.MM.yyyy}&CustomToDate={2:dd.MM.yyyy}&FromDate=CustomPeriod&Incoming=", sbankenAccountId, from, to);
driver.Navigate().GoToUrl(accountStatementDownload);
var waitExcel = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitExcel.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
string accountStatementFileName = string.Format("{0}_{1:yyyy_MM_dd}-{2:yyyy_MM_dd}.xlsx", cacheFileNamePrefix, from, to);
string accountStatementDownloadFilePath = Path.Combine(downloadFolderPath, accountStatementFileName);
// wait until file has downloaded
for (var i = 0; i < 30; i++)
{
if (File.Exists(accountStatementDownloadFilePath))
{
break;
}
Thread.Sleep(1000);
}
var length = new FileInfo(accountStatementDownloadFilePath).Length;
for (var i = 0; i < 30; i++)
{
Thread.Sleep(1000);
var newLength = new FileInfo(accountStatementDownloadFilePath).Length;
if (newLength == length && length != 0)
{
break;
}
length = newLength;
}
driver.Close();
// determine path
Console.Out.WriteLine("Successfully downloaded skandiabanken account statement excel file {0}", accountStatementDownloadFilePath);
// moving file to right place
string accountStatementDestinationPath = Path.Combine(cacheDir, accountStatementFileName);
// To copy a folder's contents to a new location:
// Create a new target folder, if necessary.
if (!Directory.Exists(cacheDir))
{
Directory.CreateDirectory(cacheDir);
}
// Move file to another location
File.Move(accountStatementDownloadFilePath, accountStatementDestinationPath);
return(accountStatementDestinationPath);
}
private static async Task <List <PayPalTransaction> > GetPayPalTransactionsListSoapAsync(IMyConfiguration configuration, TextWriter writer, DateTime from, DateTime to)
{
var payPalTransactions = new List <PayPalTransaction>();
try
{
using (var httpClient = new HttpClient(new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip
})
{
Timeout = TimeSpan.FromSeconds(30)
})
{
string payPalApiUsername = configuration.GetValue("PayPalApiUsername");
string payPalApiPassword = configuration.GetValue("PayPalApiPassword");
string payPalApiSignature = configuration.GetValue("PayPalApiSignature");
var soapEnvelopeXml = ConstructSoapEnvelope();
var doc = XDocument.Parse(soapEnvelopeXml);
var authHeader = doc.Descendants("{urn:ebay:apis:eBLBaseComponents}Credentials").FirstOrDefault();
if (authHeader != null)
{
authHeader.Element("{urn:ebay:apis:eBLBaseComponents}Username").Value = payPalApiUsername;
authHeader.Element("{urn:ebay:apis:eBLBaseComponents}Password").Value = payPalApiPassword;
authHeader.Element("{urn:ebay:apis:eBLBaseComponents}Signature").Value = payPalApiSignature;
}
var parameterHeader = doc.Descendants("{urn:ebay:api:PayPalAPI}TransactionSearchRequest").FirstOrDefault();
if (parameterHeader != null)
{
string startDate = string.Format("{0:yyyy-MM-ddTHH\\:mm\\:ssZ}", from);
string endDate = string.Format("{0:yyyy-MM-ddTHH\\:mm\\:ssZ}", to.AddDays(1));
parameterHeader.Element("{urn:ebay:api:PayPalAPI}StartDate").Value = startDate;
parameterHeader.Element("{urn:ebay:api:PayPalAPI}EndDate").Value = endDate;
}
string envelope = doc.ToString();
var request = CreateRequest(HttpMethod.Post, "https://api-3t.paypal.com/2.0/", "TransactionSearch", doc);
request.Content = new StringContent(envelope, Encoding.UTF8, "text/xml");
// request is now ready to be sent via HttpClient
//
HttpResponseMessage response = await httpClient.SendAsync(request);
if (!response.IsSuccessStatusCode)
{
throw new Exception();
}
var stream = await response.Content.ReadAsStreamAsync();
var sr = new StreamReader(stream);
var soapResponse = XDocument.Load(sr);
// parse SOAP response
var xmlTransactions = soapResponse.Descendants("{urn:ebay:apis:eBLBaseComponents}PaymentTransactions").ToList();
foreach (var xmlTransaction in xmlTransactions)
{
// build new list
var payPalTransaction = new PayPalTransaction();
//xmlTransaction.RemoveAttributes(); // trick to ignore ebl types that cannot be deserialized
//var transaction = Utils.Deserialize<PaymentTransaction>(xmlTransaction.ToString());
payPalTransaction.TransactionID = xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}TransactionID").Value;
// Converting from paypal date to date:
// 2017-08-30T21:13:37Z
// var date = DateTimeOffset.Parse(paypalTransaction.Timestamp).UtcDateTime;
payPalTransaction.Timestamp = DateTimeOffset.Parse(xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}Timestamp").Value).UtcDateTime;
payPalTransaction.Status = xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}Status").Value;
payPalTransaction.Type = xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}Type").Value;
payPalTransaction.GrossAmount = decimal.Parse(xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}GrossAmount").Value, CultureInfo.InvariantCulture);
payPalTransaction.GrossAmountCurrencyId = xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}GrossAmount").Attribute("currencyID").Value;
payPalTransaction.NetAmount = decimal.Parse(xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}NetAmount").Value, CultureInfo.InvariantCulture);
payPalTransaction.NetAmountCurrencyId = xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}NetAmount").Attribute("currencyID").Value;
payPalTransaction.FeeAmount = decimal.Parse(xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}FeeAmount").Value, CultureInfo.InvariantCulture);
payPalTransaction.FeeAmountCurrencyId = xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}FeeAmount").Attribute("currencyID").Value;
if (null != xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}Payer"))
{
payPalTransaction.Payer = xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}Payer").Value;
}
payPalTransaction.PayerDisplayName = xmlTransaction.Element("{urn:ebay:apis:eBLBaseComponents}PayerDisplayName").Value;
payPalTransactions.Add(payPalTransaction);
}
}
}
catch (System.Exception e)
{
await writer.WriteLineAsync(string.Format("ERROR: Could not get paypal transactions! '{0}'", e.Message));
}
return(payPalTransactions);
}
private async Task <List <SBankenTransaction> > GetSBankenTransactionsAsync(IMyConfiguration configuration, DateTime from, DateTime to)
{
// get SBanken configuration parameters
string clientId = configuration.GetValue("SBankenApiClientId");
string secret = configuration.GetValue("SBankenApiSecret");
string customerId = configuration.GetValue("SBankenApiCustomerId");
string accountId = configuration.GetValue("SBankenAccountId");
var sBankenTransactions = new List <SBankenTransaction>();
// see also
// https://github.com/anderaus/Sbanken.DotNet/blob/master/src/Sbanken.DotNet/Http/Connection.cs
/** Setup constants */
const string discoveryEndpoint = "https://auth.sbanken.no/identityserver";
const string apiBaseAddress = "https://api.sbanken.no";
const string bankBasePath = "/bank";
// First: get the OpenId configuration from Sbanken.
var discoClient = new DiscoveryClient(discoveryEndpoint);
var x = discoClient.Policy = new DiscoveryPolicy()
{
ValidateIssuerName = false,
};
var discoResult = await discoClient.GetAsync();
if (discoResult.Error != null)
{
throw new Exception(discoResult.Error);
}
// The application now knows how to talk to the token endpoint.
// Second: the application authenticates against the token endpoint
// ensure basic authentication RFC2617 is used
// The application must authenticate itself with Sbanken's authorization server.
// The basic authentication scheme is used here (https://tools.ietf.org/html/rfc2617#section-2 )
//var tokenClient = new TokenClient(discoResult.TokenEndpoint, clientId, secret)
//{
// BasicAuthenticationHeaderStyle = BasicAuthenticationHeaderStyle.Rfc2617
//};
var tokenClient = new TokenClient(discoResult.TokenEndpoint, clientId, secret);
var tokenResponse = await tokenClient.RequestClientCredentialsAsync();
if (tokenResponse.IsError)
{
throw new Exception(tokenResponse.ErrorDescription);
}
// The application now has an access token.
var httpClient = new HttpClient()
{
BaseAddress = new Uri(apiBaseAddress),
DefaultRequestHeaders =
{
{ "customerId", customerId }
}
};
// Finally: Set the access token on the connecting client.
// It will be used with all requests against the API endpoints.
httpClient.SetBearerToken(tokenResponse.AccessToken);
// retrieve the customer's transactions
// RFC3339 / ISO8601 with 3 decimal places
// yyyy-MM-ddTHH:mm:ss.fffK
string querySuffix = string.Format(CultureInfo.InvariantCulture, "?length=1000&startDate={0:yyyy-MM-ddTHH:mm:ss.fffK}&endDate={1:yyyy-MM-ddTHH:mm:ss.fffK}", from, to);
var transactionResponse = await httpClient.GetAsync($"{bankBasePath}/api/v1/Transactions/{accountId}{querySuffix}");
var transactionResult = await transactionResponse.Content.ReadAsStringAsync();
// parse json
dynamic jsonDe = JsonConvert.DeserializeObject(transactionResult);
if (jsonDe != null)
{
foreach (var transaction in jsonDe.items)
{
var amount = transaction.amount;
var text = transaction.text;
var transactionType = transaction.transactionType;
var transactionTypeText = transaction.transactionTypeText;
var accountingDate = transaction.accountingDate;
var interestDate = transaction.interestDate;
var transactionId = transaction.transactionId;
// Note, until Sbanken fixed their unique transaction Id issue, generate one ourselves
if (transactionId == null || !transactionId.HasValues || transactionId == JTokenType.Null)
{
// ensure we are working with proper objects
DateTime localAccountingDate = accountingDate;
DateTime localInterestDate = interestDate;
decimal localAmount = amount;
// convert to string
string accountingDateString = localAccountingDate.ToString("G", CultureInfo.InvariantCulture);
string interestDateString = localInterestDate.ToString("G", CultureInfo.InvariantCulture);
string amountString = localAmount.ToString("G", CultureInfo.InvariantCulture);
// combine and create md5
string uniqueContent = $"{accountingDateString}{interestDateString}{transactionTypeText}{text}{amountString}";
string hashCode = Utils.CreateMD5(uniqueContent);
transactionId = hashCode;
}
var sBankenTransaction = new SBankenTransaction();
sBankenTransaction.TransactionDate = accountingDate;
sBankenTransaction.InterestDate = interestDate;
sBankenTransaction.ArchiveReference = transactionId;
sBankenTransaction.Type = transactionTypeText;
sBankenTransaction.Text = text;
var date = new Date();
var currentDate = date.CurrentDate;
// check if card details was specified
if ((bool)transaction.cardDetailsSpecified)
{
var cardDatailsCardNumber = transaction.cardDetails.cardNumber;
var cardDatailsCurrencyAmount = transaction.cardDetails.currencyAmount;
var cardDatailsCurrencyRate = transaction.cardDetails.currencyRate;
var cardDatailsCurrencyCode = transaction.cardDetails.originalCurrencyCode;
var cardDatailsMerchantCategoryCode = transaction.cardDetails.merchantCategoryCode;
var cardDatailsMerchantName = transaction.cardDetails.merchantName;
var cardDatailsPurchaseDate = transaction.cardDetails.purchaseDate;
var cardDatailsTransactionId = transaction.cardDetails.transactionId;
sBankenTransaction.ExternalPurchaseDate = cardDatailsPurchaseDate;
sBankenTransaction.ExternalPurchaseAmount = cardDatailsCurrencyAmount;
sBankenTransaction.ExternalPurchaseCurrency = cardDatailsCurrencyCode;
sBankenTransaction.ExternalPurchaseVendor = cardDatailsMerchantName;
sBankenTransaction.ExternalPurchaseExchangeRate = cardDatailsCurrencyRate;
// NOTE! fix a likely bug in the API where the external purchase date is the wrong year
if (sBankenTransaction.ExternalPurchaseDate.Year == currentDate.Year &&
sBankenTransaction.ExternalPurchaseDate.Month > currentDate.Month)
{
sBankenTransaction.ExternalPurchaseDate = sBankenTransaction.ExternalPurchaseDate.AddYears(-1);
}
}
// set account change
sBankenTransaction.AccountChange = amount;
if (amount > 0)
{
sBankenTransaction.AccountingType = SBankenTransaction.AccountingTypeEnum.IncomeUnknown;
}
else
{
sBankenTransaction.AccountingType = SBankenTransaction.AccountingTypeEnum.CostUnknown;
}
if (transactionId != null && transactionId != "")
{
sBankenTransactions.Add(sBankenTransaction);
}
}
}
return(sBankenTransactions);
}
public static List <OberloOrder> ScrapeOberloOrders(IMyConfiguration configuration, DateTime from, DateTime to)
{
var oberloOrders = new List <OberloOrder>();
string userDataDir = configuration.GetValue("UserDataDir");
string oberloUsername = configuration.GetValue("OberloUsername");
string oberloPassword = configuration.GetValue("OberloPassword");
string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath");
var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath);
// https://app.oberlo.com/orders?from=2017-01-01&to=2017-12-31&page=1
int page = 1;
string url = string.Format("https://app.oberlo.com/orders?from={0:yyyy-MM-dd}&to={1:yyyy-MM-dd}&page={2}", from, to, page);
driver.Navigate().GoToUrl(url);
var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
var ready = wait.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
// login if login form is present
if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@name='email']")) &&
SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@name='password']")))
{
IWebElement username = driver.FindElement(By.XPath("//input[@name='email']"));
IWebElement password = driver.FindElement(By.XPath("//input[@name='password']"));
username.Clear();
username.SendKeys(oberloUsername);
// if the above crash, it might very well be Chrome and ChromeDriver incompatibility
// please update the ChromeDriver to fit the Chrome version used.
password.Clear();
password.SendKeys(oberloPassword);
// use password field to submit form
password.Submit();
var wait2 = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
var ready2 = wait2.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
}
IJavaScriptExecutor js = driver as IJavaScriptExecutor;
var json = js.ExecuteScript("return window.App.payload.orders;");
// convert to json dynamic object
string jsonString = JsonConvert.SerializeObject(json);
dynamic jsonD = JsonConvert.DeserializeObject(jsonString);
// https://app.oberlo.com/orders?from=2017-01-01&to=2017-12-31&page=1
// identify how many pages on order page
// current_page
// last_page
// per_page
// data (System.Collections.ObjectModel)
int current_page = jsonD.current_page;
int last_page = jsonD.last_page;
int per_page = jsonD.per_page;
// process orders on page
var orders = jsonD.data;
foreach (var order in orders)
{
// order_name
// processed_at
// total_price
// shipping_name
// shipping_zip
// shipping_city
// shipping_address1
// orderitems
var orderName = order.order_name;
var processedAt = order.processed_at;
var totalPrice = order.total_price;
var financialStatus = order.financial_status;
var fulfillmentStatus = order.fulfillment_status;
var shippingName = order.shipping_name;
var shippingZip = order.shipping_zip;
var shippingCity = order.shipping_city;
var shippingAddress1 = order.shipping_address1;
var shippingAddress2 = order.shipping_address2;
var orderNote = order.local_note;
var orderitems = order.orderitems;
foreach (var orderitem in orderitems)
{
var aliOrderNumber = orderitem.ali_order_no;
var SKU = orderitem.sku;
var supplier = orderitem.supplier_name;
var productName = orderitem.title;
var variant = orderitem.variant_title;
var cost = orderitem.cost;
var quantity = orderitem.quantity;
var price = orderitem.price;
string trackingNumber = "";
foreach (var fulfillment in orderitem.fulfillments)
{
if (trackingNumber.Equals(""))
{
trackingNumber = fulfillment.tracking_number;
}
else
{
trackingNumber += ", " + fulfillment.tracking_number;
}
}
var oberloOrder = new OberloOrder();
oberloOrder.OrderNumber = orderName;
oberloOrder.CreatedDate = processedAt;
oberloOrder.FinancialStatus = financialStatus;
oberloOrder.FulfillmentStatus = fulfillmentStatus;
oberloOrder.Supplier = supplier;
oberloOrder.SKU = SKU;
oberloOrder.ProductName = productName;
oberloOrder.Variant = variant;
oberloOrder.Quantity = quantity;
oberloOrder.ProductPrice = price;
oberloOrder.TrackingNumber = trackingNumber;
oberloOrder.AliOrderNumber = aliOrderNumber;
oberloOrder.CustomerName = shippingName;
oberloOrder.CustomerAddress = shippingAddress1;
oberloOrder.CustomerAddress2 = shippingAddress2;
oberloOrder.CustomerCity = shippingCity;
oberloOrder.CustomerZip = shippingZip;
oberloOrder.OrderNote = orderNote;
//oberloOrder.OrderState = orderState;
oberloOrder.TotalPrice = totalPrice;
oberloOrder.Cost = cost;
oberloOrders.Add(oberloOrder);
}
}
// and process the rest of the pages
for (int i = current_page + 1; i <= last_page; i++)
{
ScrapeOberloOrders(driver, oberloOrders, from, to, i);
}
driver.Close();
return(oberloOrders);
}
public static List <AliExpressOrder> ScrapeAliExpressOrders(IMyConfiguration configuration, DateTime from)
{
string userDataDir = configuration.GetValue("UserDataDir");
string cacheDir = configuration.GetValue("CacheDir");
string aliExpressUsername = configuration.GetValue("AliExpressUsername");
string aliExpressPassword = configuration.GetValue("AliExpressPassword");
string chromeDriverExePath = configuration.GetValue("ChromeDriverExePath");
var aliExpressOrders = new List <AliExpressOrder>();
// http://blog.hanxiaogang.com/2017-07-29-aliexpress/
var driver = Utils.GetChromeWebDriver(userDataDir, chromeDriverExePath);
driver.Navigate().GoToUrl("https://login.aliexpress.com");
var waitLoginPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitLoginPage.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
// login if login form is present
// https://philipcaande.wordpress.com/2016/01/17/iframe-in-selenium-c/
if (SeleniumUtils.IsElementPresent(driver, By.Id("alibaba-login-box")))
{
// switch to login iframe
IWebElement iframe = driver.FindElement(By.Id("alibaba-login-box"));
driver.SwitchTo().Frame(iframe);
// login if login form is present
if (SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='fm-login-id']")) &&
SeleniumUtils.IsElementPresent(driver, By.XPath("//input[@id='fm-login-password']")))
{
IWebElement username = driver.FindElement(By.XPath("//input[@id='fm-login-id']"));
IWebElement password = driver.FindElement(By.XPath("//input[@id='fm-login-password']"));
username.Clear();
username.SendKeys(aliExpressUsername);
password.Clear();
password.SendKeys(aliExpressPassword);
// use password field to submit form
password.Submit();
var waitLoginIFrame = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitLoginIFrame.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));
}
// switch back to main frame
driver.SwitchTo().DefaultContent();
}
try
{
var waitMainPage = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
waitMainPage.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.UrlToBe("https://www.aliexpress.com/"));
}
catch (WebDriverTimeoutException)
{
Console.WriteLine("Timeout - Logged in to AliExpress to late. Stopping.");
return(aliExpressOrders);
}
// go to order list
driver.Navigate().GoToUrl("https://trade.aliexpress.com/orderList.htm");
// identify how many pages on order page (1/20)
var tuple = GetAliExpressOrderPageNumber(driver);
int curPage = tuple.Item1;
int numPages = tuple.Item2;
Console.WriteLine("Found {0} Pages", numPages);
// scrape one and one page
for (int i = 1; i <= numPages; i++)
{
// if this method returns false, it means we have reached the from date
if (!ScrapeAliExpressOrderPage(aliExpressOrders, driver, i, from))
{
break;
}
}
driver.Close();
return(aliExpressOrders);
}