} // FetchBackdoorData
/// <summary>
/// Constructs the Harvester object. The object is not ready to harvest. Call to Init() first.
/// </summary>
public Harvester(AccountData oAccountData, ASafeLog log) : base(log)
{
ErrorsToEmail = new SortedDictionary <string, string>();
AccountData = oAccountData;
VerboseLogging = false;
Hopper = new Hopper(VatReturnSourceType.Linked);
IsLoggedIn = false;
} // constructor
public static void SetBackdoorData(int nCustomerMarketplaceID, Hopper oHopper)
{
if (oHopper == null)
{
return;
}
lock (typeof(Harvester)) {
if (ms_oBackdoorData == null)
{
ms_oBackdoorData = new SortedDictionary <int, Hopper>();
}
ms_oBackdoorData[nCustomerMarketplaceID] = oHopper;
} // lock
} // SetBackdoorData
} // ForEachFile
public void FetchBackdoorData(Hopper oHopper)
{
lock (this) {
Clear();
if (oHopper == null)
{
return;
}
Source = oHopper.Source;
ErrorCount = oHopper.ErrorCount;
var aryDataTypes = (DataType[])Enum.GetValues(typeof(DataType));
var oFileTypes = new List <FileType>((FileType[])Enum.GetValues(typeof(FileType)));
oFileTypes.Remove(FileType.Unknown);
foreach (DataType dt in aryDataTypes)
{
foreach (FileType ft in oFileTypes)
{
foreach (KeyValuePair <string, HarvesterError> pair in oHopper.Errors[dt][ft])
{
Errors[dt][ft][pair.Key] = pair.Value;
}
foreach (KeyValuePair <string, byte[]> pair in oHopper.Files[dt][ft])
{
Files[dt][ft][pair.Key] = pair.Value;
}
} // for each file type
foreach (KeyValuePair <string, ISeeds> pair in oHopper.Seeds[dt])
{
Seeds[dt][pair.Key] = pair.Value;
}
} // for each data type
} // lock
} // FetchBackdoorData
} // ExtractTaxOfficeNumber
private void FetchRtiTaxYears()
{
if (string.IsNullOrWhiteSpace(TaxOfficeNumber))
{
Debug("Not fetching RTI Tax Years: Tax Office number is empty.");
return;
} // if
Debug("Fetching RTI Tax Years started...");
HtmlDocument doc = GetPage("/paye/org/" + TaxOfficeNumber + "/account");
if ((doc == null) || (doc.DocumentNode == null))
{
throw new HarvesterException("Failed to fetch PAYE account page.");
}
var oOutput = new MemoryStream();
doc.Save(oOutput);
var smd = new SheafMetaData {
BaseFileName = "PAYE RTI Tax Year",
DataType = DataType.PayeRtiTaxYears,
FileType = FileType.Html,
Thrasher = null
};
Hopper.Add(smd, oOutput.ToArray());
HtmlNode oTHead = doc.DocumentNode.SelectSingleNode("//*[@id=\"top\"]/div[3]/div[2]/div/div[2]/table[1]/thead");
if (oTHead == null)
{
Info("RTI tax years table head not found.");
return;
} // if
HtmlNodeCollection oHeadRows = oTHead.SelectNodes("tr");
if ((oHeadRows == null) || (oHeadRows.Count != 1))
{
throw new HarvesterException("RTI tax years table head is empty.");
}
HtmlNodeCollection oHeadCells = oHeadRows[0].SelectNodes("th | td");
string[] aryExpectedColumnHeaders = new [] {
"Date",
"Amount paid in period",
"Amount due in period",
};
if ((oHeadCells == null) || (oHeadCells.Count != aryExpectedColumnHeaders.Length))
{
throw new HarvesterException(string.Format("Failed to fetch RTI tax years: no cells in header row"));
}
for (int i = 0; i < aryExpectedColumnHeaders.Length; i++)
{
if (!oHeadCells[i].InnerText.Trim().StartsWith(aryExpectedColumnHeaders[i]))
{
Info(
"Not fetching RTI tax years: unexpected column {0} name: {1} (expected: {2})",
i, oHeadCells[i].InnerText, aryExpectedColumnHeaders[i]
);
return;
} // if
} // for
HtmlNode oTBody = doc.DocumentNode.SelectSingleNode("//*[@id=\"top\"]/div[3]/div[2]/div/div[2]/table[1]/tbody");
if (oTBody == null)
{
throw new HarvesterException("RTI tax years table body not found.");
}
HtmlNodeCollection oRows = oTBody.SelectNodes("tr");
if ((oRows == null) || (oRows.Count < 1))
{
throw new HarvesterException("RTI tax years data not found.");
}
bool bFirst = true;
int nRowNum = -1;
int nFirstYear = 0;
int nLastYear = 0;
var data = new List <RtiTaxYearRowData>();
foreach (HtmlNode oTR in oRows)
{
nRowNum++;
HtmlNodeCollection oCells = oTR.SelectNodes("th | td");
if ((oCells == null) || (oCells.Count < 1))
{
throw new HarvesterException(string.Format(
"Failed to fetch RTI tax years: no cells in row {0}.",
nRowNum
));
} // if
if (bFirst)
{
bFirst = false;
HtmlNode oCell = oCells[0];
if (!oCell.Attributes.Contains("colspan") || (oCell.Attributes["colspan"].Value != "3"))
{
throw new HarvesterException(string.Format(
"Failed to fetch RTI tax years: incorrect format in row {0}",
nRowNum
));
} // if
if (oCell.InnerText.Trim() == "Previous tax years")
{
break;
}
MatchCollection match = Regex.Matches(oCell.InnerText.Trim(), @"^Current tax year (\d\d)(\d\d)-(\d\d)$");
if (match.Count != 1)
{
throw new HarvesterException(string.Format(
"Failed to fetch RTI tax years: incorrect content in row {0}.",
nRowNum
));
} // if
GroupCollection grp = match[0].Groups;
if (grp.Count != 4)
{
throw new HarvesterException(string.Format(
"Failed to fetch RTI tax years: unexpected content in row {0}.",
nRowNum
));
} // if
nFirstYear = Convert.ToInt32(grp[1].Value) * 100 + Convert.ToInt32(grp[2].Value);
nLastYear = Convert.ToInt32(grp[1].Value) * 100 + Convert.ToInt32(grp[3].Value);
Info("Current tax year: {0} - {1}", nFirstYear, nLastYear);
continue;
} // if first row
string sFirstCell = oCells.Count > 0 ? oCells[0].InnerText.Trim() : string.Empty;
if (oCells.Count != 3)
{
if ((oCells.Count == 1) && (sFirstCell == "Previous tax years"))
{
break;
}
throw new HarvesterException(string.Format(
"Failed to fetch RTI tax years: unexpected number of cells in row {0}.",
nRowNum
));
} // if
if (sFirstCell == "Total")
{
break;
}
try {
data.Add(new RtiTaxYearRowData(sFirstCell, oCells[1].InnerText.Trim(), oCells[2].InnerText.Trim()));
} catch (Exception e) {
throw new HarvesterException(
string.Format(
"Failed to fetch RTI tax years: unexpected format in row {0}.",
nRowNum
),
e
);
} // try
} // for each row
int nCurYear = nFirstYear;
var rtys = new RtiTaxYearSeeds();
foreach (RtiTaxYearRowData rd in data.ToArray().Reverse())
{
rtys.Months.Add(new RtiTaxMonthSeed {
DateStart = new DateTime(nCurYear, rd.MonthStart, rd.DayStart),
DateEnd = new DateTime(nCurYear, rd.MonthEnd, rd.DayEnd),
AmountPaid = new Coin(rd.AmountPaid, "GBP"),
AmountDue = new Coin(rd.AmountDue, "GBP")
});
if (rd.MonthStart == 12)
{
nCurYear = nLastYear;
}
} // for each
Hopper.Add(smd, rtys);
Debug("Fetching RTI Tax Years complete.");
} // FetchRtiTaxYears
} // Load
/// <summary>
/// Fetches one file from the Field and stores it in the Hopper as byte[].
/// Adds HarvesterError to the Hopper in case of error.
/// </summary>
/// <param name="task">HTTP request result.</param>
private void GetFile(Task <HttpResponseMessage> task)
{
SheafMetaData fi;
HttpResponseMessage response = task.Result;
var sUrl = response.RequestMessage.RequestUri.PathAndQuery;
lock (this.vatReturnsMetaDataLock) {
fi = this.m_oVatReturnsMetaData[sUrl];
} // lock
Info("GetFile: retrieving {0}", response.RequestMessage);
if (!response.IsSuccessStatusCode)
{
string sErrMsg = response.StatusCode.ToString() + ": " + response.ReasonPhrase;
Error("Not saving because of error. Status code {0}", sErrMsg);
Hopper.Add(fi, response);
lock (ErrorsToEmail) {
ErrorsToEmail[sUrl] = sErrMsg;
} // lock
return;
} // if
Stream oInputStream = response.Content.ReadAsStreamAsync().Result;
var oOutput = new MemoryStream();
var outputFile = new BinaryWriter(oOutput);
const int nBufSize = 8192;
var buf = new byte[nBufSize];
int nRead = oInputStream.Read(buf, 0, nBufSize);
while (nRead > 0)
{
outputFile.Write(buf, 0, nRead);
nRead = oInputStream.Read(buf, 0, nBufSize);
} // while
outputFile.Close();
byte[] oFile = oOutput.ToArray();
Hopper.Add(fi, oFile);
if (fi.Thrasher != null)
{
ISeeds x = fi.Thrasher.Run(fi, oFile);
if (x == null)
{
var vrs = (VatReturnSeeds)fi.Thrasher.Seeds;
lock (ErrorsToEmail) {
ErrorsToEmail[sUrl] = vrs.FatalError;
} // lock
} // if
Hopper.Add(fi, x);
} // if
} // GetFile
} // Run
/// <summary>
/// Main harvest function. Logs in to hmrc.gov.uk and fetches data.
/// </summary>
/// <param name="bValidateCredentialsOnly">true to validate credentials only,
/// false to login and download data.</param>
/// <param name="nCustomerMarketplaceID">Customer marketplace id for fetching back-door data.</param>
public virtual void Run(bool bValidateCredentialsOnly, int nCustomerMarketplaceID)
{
Debug(
"Harvester run mode: {0}.",
bValidateCredentialsOnly ? "validate credentials only" : "login and download data"
);
try {
if (!bValidateCredentialsOnly)
{
Hopper oBackdoorData = FetchBackdoorData(nCustomerMarketplaceID, this);
if (oBackdoorData != null)
{
Hopper.FetchBackdoorData(oBackdoorData);
Debug("Harvester running is complete.");
return;
} // if
if (Password == VendorInfo.TopSecret)
{
if (0 < ObjectFactory.GetInstance <IUsersRepository>().ExternalUserCount(UserName))
{
Debug(
"This HMRC account for customer {0} was created from uploaded files, nothing to retrieve.",
UserName
);
return;
} // if HMRC login is customer's email
} // if the password is...
} // if do retrieve data
} catch (Exception e) {
throw new ApiException(e.Message, e);
} // try
try {
if (!IsLoggedIn)
{
Login(GetLoginRequestDetails(GetPage("")));
}
} catch (InvalidCredentialsException) {
throw;
} catch (Exception e) {
throw new ApiException(e.Message, e);
} // try
if (!IsLoggedIn)
{
throw new ApiException("Not logged in.");
}
if (bValidateCredentialsOnly)
{
Debug("Harvester running is complete.");
return;
} // if
try {
string sUserVatID = GetUserVatID();
Debug("Harvester has validated login credentials.");
Debug("Harvester starts downloading data.");
VatReturns(sUserVatID);
FetchRtiTaxYears();
Debug("Harvester running is complete.");
} catch (Exception e) {
Alert(e, "Exception caught during retrieving HMRC data.");
throw new ApiException(e.Message, e);
} // try
} // Run