public static void UnloadKey(SshConnectionData sftpConnectionData, string clientName, string authorizedKeyPath)
{
lock (locker)
{
// Inserire quì la connessione ssh e salvataggio della chiave pubblica sul server
// Lo faccio tramite SSH, in questo modo è possibile tenere webserver e server ssh anche separati
SftpHelper sftp = new SftpHelper();
// download authorized_keys
sftp.DownloadFile(sftpConnectionData, authorizedKeyPath, Constants.TEMP_AUTHORIZED_KEYS_FILENAME);
// check if authorized keys already contains the client key
bool clientCurrentlyLoaded = IsClientCurrentlyLoaded(clientName);
// aggiunta della nuova chiave ad authorized_keys
if (clientCurrentlyLoaded)
{
Console.WriteLine($"Unloading {clientName} key.");
RemoveAuthorizedKey(clientName);
// upload di authorized_keys
sftp.UploadFile(sftpConnectionData, Constants.TEMP_AUTHORIZED_KEYS_FILENAME, authorizedKeyPath);
}
}
}
private static UploadResult UploadExportCsvToSftp(string remoteDirectory, string uploadFileNamePath)
{
var result = new UploadResult
{
SourceFileNamePath = uploadFileNamePath,
DestinationFileNamePath = Path.Combine(remoteDirectory, Path.GetFileName(uploadFileNamePath))
};
if (SftpHelper.CreateDirectory(remoteDirectory))
{
if (SftpHelper.UploadFile(result.DestinationFileNamePath, uploadFileNamePath, out string errorMessage))
{
result.Success = true;
result.SuccessMessage = "The upload has completed successfully";
}
else
{
result.Success = false;
result.ErrorMessage = errorMessage;
}
}
else
{
result.Success = false;
result.ErrorMessage = "Unable to create upload folder in remote server";
}
return(result);
}
public static void SaveKeys(SshConnectionData sftpConnectionData, string sshUsername, string clientName, string publicKey, string authorizedKeyPath, int?forwardingPort)
{
lock (locker)
{
// Inserire quì la connessione ssh e salvataggio della chiave pubblica sul server
// Lo faccio tramite SSH, in questo modo è possibile tenere webserver e server ssh anche separati
SftpHelper sftp = new SftpHelper();
// download authorized_keys
sftp.DownloadFile(sftpConnectionData, authorizedKeyPath, Constants.TEMP_AUTHORIZED_KEYS_FILENAME);
#warning la verifica della presenza funziona, ma non è un metodo efficiente. forse andrebbe fatto a database prima di scaricare il file delle auth
// check if authorized keys already contains the client key
bool clientAlreadyEnabled = IsClientAlreadyEnabled(publicKey, clientName);
// aggiunta della nuova chiave ad authorized_keys
if (!clientAlreadyEnabled)
{
AddKeyToAuthorized(publicKey, clientName, forwardingPort);
// upload di authorized_keys
sftp.UploadFile(sftpConnectionData, Constants.TEMP_AUTHORIZED_KEYS_FILENAME, authorizedKeyPath);
}
}
}
public bool Init(Uri uri)
{
using (new SBLicenseManager.TElSBLicenseManager
{
LicenseKey = "8F1317CD48AC4E68BABA5E339D8B365414D7ADA0289CA037E9074D29AD95FF3EC5D796BEFF0FBADB3BD82F48644C9EB810D9B5A305E0D2A1885C874D8BF974B9608CE918113FBE2AA5EEF8264C93B25ABEA98715DB4AD265F47CE02FC9952D69F2C3530B6ABAAA4C43B45E7EF6A8A0646DA038E34FBFB629C2BF0E83C6B348726E622EBD52CA05CF74C68F1279849CCD0C13EA673916BA42684015D658B8E7626F15BD826A4340EDB36CE55791A051FDBCF9FA1456C3B5008AD9990A0185C0EA3B19F9938CB7DA1FE82736ED4C7A566D4BFD53411E8380F4B020CB50E762520EFAE190836FD253B00DB18D4A485C7DC918AA4DCEC856331DD231CC8DC9C741C3"
})
{
var host = string.Format(@"sftp://{0}", uri.Host);
var privateKeyFile = HttpUtility.ParseQueryString(uri.Query).Get(PrivateKeyFilePathParameter);
try
{
sftpClient = !string.IsNullOrWhiteSpace(privateKeyFile)
? SftpHelper.CreateClient(host, privateKeyFile, String.Empty, uri.UserInfo)
: SftpHelper.CreateClient(host);
}
catch (Exception e)
{
TraceSource.TraceError("Error setting up sftpClient: Host {0}: Error {1}", host, e.Message);
return(false);
}
return(sftpClient != null);
}
}
protected override void Process()
{
CurrentLanguage = PrimaryLanguage;
var Soap = new AssortmentService();
#if CACHE
if (!File.Exists(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID)))
{
ImagesXml = XDocument.Parse(Soap.GetAssortmentImages(Connector.ConnectorID));
ImagesXml.Save(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID));
}
else
{
ImagesXml = XDocument.Load(String.Format(ImagesCacheFile, Connector.ConnectorID, PrimaryLanguage.LanguageID));
}
#else
ImagesXml = XDocument.Parse(Soap.GetAssortmentImages(Connector.ConnectorID));
#endif
SBLicenseManager.TElSBLicenseManager m = new SBLicenseManager.TElSBLicenseManager();
m.LicenseKey = "8F1317CD48AC4E68BABA5E339D8B365414D7ADA0289CA037E9074D29AD95FF3EC5D796BEFF0FBADB3BD82F48644C9EB810D9B5A305E0D2A1885C874D8BF974B9608CE918113FBE2AA5EEF8264C93B25ABEA98715DB4AD265F47CE02FC9952D69F2C3530B6ABAAA4C43B45E7EF6A8A0646DA038E34FBFB629C2BF0E83C6B348726E622EBD52CA05CF74C68F1279849CCD0C13EA673916BA42684015D658B8E7626F15BD826A4340EDB36CE55791A051FDBCF9FA1456C3B5008AD9990A0185C0EA3B19F9938CB7DA1FE82736ED4C7A566D4BFD53411E8380F4B020CB50E762520EFAE190836FD253B00DB18D4A485C7DC918AA4DCEC856331DD231CC8DC9C741C3";
FtpInfo ftpInfo = SftpHelper.GetFtpInfo(Connector);
Logger.DebugFormat("Exporting images for language '{0}'", CurrentLanguage.Language.Name);
SortedDictionary <string, catalog_product_entity> currentProducts;
SortedDictionary <string, eav_attribute> attributeList;
using (var helper = new MagentoMySqlHelper(Connector.Connection))
{
attributeList = helper.GetAttributeList(PRODUCT_ENTITY_TYPE_ID);
currentProducts = helper.GetSkuList();
}
var products = (from i in ImagesXml.Root.Elements("Products").Elements("ProductMedia")
where i.Attribute("ManufacturerID") != null
orderby Convert.ToInt32(i.Attribute("ProductID").Value) descending
group i by i.Attribute("ManufacturerID").Value into grouped
select grouped).ToList();
var directoryStructureRecords = (from i in ImagesXml.Root.Elements("Products").Elements("ProductMedia")
// this catches the changes in webservice structure
let urlAttribute = i.Attribute("Url")
let uri = (urlAttribute != null) ? new Uri(urlAttribute.Value) : new Uri(i.Value)
select new
{
FirstLevel = uri.Segments.Last().Substring(0, 1),
SecondLevel = uri.Segments.Last().Substring(1, 1)
}).GroupBy(x => x.FirstLevel).ToDictionary(x => x.Key, y => y.Select(z => z.SecondLevel).Distinct());
var dirClient = SftpHelper.GetSFTPClient(ftpInfo);
foreach (var kv in directoryStructureRecords)
{
SftpHelper.EnsurePath(dirClient, ftpInfo.FtpPath + kv.Key);
//if (!dirClient.Exists(kv.Key))
// dirClient.MakeDirectory(kv.Key);
foreach (var sl in kv.Value)
{
//if (!dirClient.Exists(kv.Key + "/" + sl))
// dirClient.MakeDirectory(kv.Key + "/" + sl);
SftpHelper.EnsurePath(dirClient, ftpInfo.FtpPath + kv.Key + "/" + sl);
}
}
dirClient.Close(true);
int totalImages = products.Select(x => x.Count()).Sum();
int totalRecords = products.Count;
int totalProcessed = 0;
if (totalRecords == 0)
{
Logger.DebugFormat("Finished exporting images for language '{0}'", CurrentLanguage.Language.Name);
return;
}
ParallelOptions options = new ParallelOptions()
{
MaxDegreeOfParallelism = 8
};
Parallel.ForEach(Partitioner.Create(0, totalRecords), options, (range, loopState) =>
{
TElSimpleSFTPClient client = SftpHelper.GetSFTPClient(ftpInfo);
try
{
using (var helper = new AssortmentHelper(Connector.Connection, Version))
{
for (int index = range.Item1; index < range.Item2; index++)
{
var product = products[index];
var sku = product.Key.Trim();
catalog_product_entity entity;
if (!currentProducts.TryGetValue(sku, out entity))
{
continue;
}
List <string> currentPaths = new List <string>();
//helper.ResetGallery(entity, 0);
var sequencedImages = product.OrderBy(x => Convert.ToInt32(x.Attribute("Sequence").Value)).ToList();
var thumbnailImage = sequencedImages.FirstOrDefault(c => c.Attribute("IsThumbnailImage") != null && Convert.ToBoolean(c.Attribute("IsThumbnailImage").Value));
int?thumbImageIdx = null;
if (thumbnailImage != null)
{
thumbImageIdx = sequencedImages.IndexOf(thumbnailImage);
}
for (int idx = 0; idx < sequencedImages.Count; idx++)
{
var image = sequencedImages[idx];
int position = Convert.ToInt32(image.Attribute("Sequence").Value) + 1;
int storeid = 0;
// this catches the changes in webservice structure
var sourceUri = (image.Attribute("Url") != null) ? new Uri(image.Attribute("Url").Value) : new Uri(image.Value);
var label = image.Attribute("Description") != null ? image.Attribute("Description").Value : string.Empty;
string fileName = sourceUri.Segments.Last();
string linuxPath = String.Format(@"/{0}/{1}/{2}", fileName.Substring(0, 1),
fileName.Substring(1, 1),
fileName.Replace(" ", ""));
#if DEBUG
//sourceUri = new Uri(sourceUri.AbsoluteUri.Replace("localhost", "10.172.26.1"));
sourceUri = new Uri(sourceUri.AbsoluteUri.Replace("localhost", "172.16.250.94"));
#endif
string path = DownloadImage(sourceUri, client, ftpInfo.FtpPath, Logger);
if (path != null)
{
helper.AddImageToGallery(entity.entity_id, storeid, path, position, label, Logger);
if (idx == 0)
{
helper.SyncAttributeValue(attributeList["image"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, path);
helper.SyncAttributeValue(attributeList["small_image"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, path);
helper.SyncAttributeValue(attributeList["image_label"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, label);
}
if (thumbImageIdx.HasValue && thumbImageIdx.Value == idx)
{
helper.SyncAttributeValue(attributeList["thumbnail"].attribute_id, PRODUCT_ENTITY_TYPE_ID, 0, entity.entity_id, path);
}
currentPaths.Add(path);
}
}
var currentImages = helper.GetGalleryItems(entity);
CleanupGallery(helper, entity, currentImages, currentPaths, client, ftpInfo.FtpPath);
Interlocked.Increment(ref totalProcessed);
//Logger.DebugFormat("Connections : {0}", ftp.ConnectionOpen);
//Logger.DebugFormat("Processed {0}", totalProcessed);
if (totalProcessed % 100 == 0)
{
Logger.DebugFormat(String.Format("Processed {0} of {1} products", totalProcessed, totalRecords));
}
}
}
}
catch (Exception e)
{
Logger.Error("Image uploader error ", e);
}
});
Logger.DebugFormat("Finished exporting images for language '{0}'", CurrentLanguage.Language.Name);
}
public void EnsurePathOnRemote(string remotePath)
{
SftpHelper.EnsurePath(Client, remotePath);
}
public SftpMediaExchangeProvider(Connector connector)
{
Client = SftpHelper.GetSFTPClient(SftpHelper.GetFtpInfo(connector));
}
public IFtpHelper GetFtpHelper(ILog log)
{
var ftpHelper = new SftpHelper(log, this.Host, this.Port, this.Username, this.Password);
return(ftpHelper);
}
protected override void Process()
{
SBLicenseManager.TElSBLicenseManager m = new SBLicenseManager.TElSBLicenseManager();
m.LicenseKey = "8F1317CD48AC4E68BABA5E339D8B365414D7ADA0289CA037E9074D29AD95FF3EC5D796BEFF0FBADB3BD82F48644C9EB810D9B5A305E0D2A1885C874D8BF974B9608CE918113FBE2AA5EEF8264C93B25ABEA98715DB4AD265F47CE02FC9952D69F2C3530B6ABAAA4C43B45E7EF6A8A0646DA038E34FBFB629C2BF0E83C6B348726E622EBD52CA05CF74C68F1279849CCD0C13EA673916BA42684015D658B8E7626F15BD826A4340EDB36CE55791A051FDBCF9FA1456C3B5008AD9990A0185C0EA3B19F9938CB7DA1FE82736ED4C7A566D4BFD53411E8380F4B020CB50E762520EFAE190836FD253B00DB18D4A485C7DC918AA4DCEC856331DD231CC8DC9C741C3";
_monitoring.Notify(Name, 0);
using (var unit = GetUnitOfWork())
{
foreach (Connector connector in base.Connectors.Where(c => ((ConnectorType)c.ConnectorType).Has(ConnectorType.WebAssortment)))
{
#if DEBUG
if (connector.ConnectorID != 5)
{
continue;
}
#endif
_monitoring.Notify(Name, connector.ConnectorID);
SetSoldenPeriod(connector);
string singleProduct = string.Empty;
try
{
log.DebugFormat("Start Magento Assortment Export for {0}", connector.Name);
if (connector.ConnectorSystemType == null)
{
log.AuditError(string.Format("No Connector System Settings found for {0}, Magento Export can not be executed!", connector.Name), "Magento Export");
continue;
}
DateTime start = DateTime.Now;
MagentoExporter exporter = new MagentoExporter(connector, log, @"D:\magento");
exporter.Execute();
CustomerExporter cExporter = new CustomerExporter(connector, log, GetConfiguration());
cExporter.Execute();
log.DebugFormat("Finished Magento Export For {0}", connector.Name);
}
catch (OutOfMemoryException ex)
{
log.AuditCritical("Magento export -> Out of memory exception for " + connector.ConnectorID, ex, "Magento export");
_monitoring.Notify(Name, -1);
}
catch (Exception ex)
{
log.AuditError("Error in Magento Plugin", ex);
_monitoring.Notify(Name, -2);
}
var triggerIndex = connector.ConnectorSettings.GetValueByKey <bool>("TriggerIndexing", false);
if (triggerIndex)
{
log.Info("Will place trigger for indexing");
try
{
var info = SftpHelper.GetFtpTriggerIndexInfo(connector);
IndexerHelper hlp = new IndexerHelper(info, "cache");
hlp.CreateAssortmentTrigger();
log.Info("Placed a trigger file");
}
catch (Exception e)
{
log.AuditError("Couldnt upload a trigger file", e, "Magento export");
_monitoring.Notify(Name, -3);
}
}
}
_monitoring.Notify(Name, 1);
}
}
protected override void Process()
{
SBLicenseManager.TElSBLicenseManager m = new SBLicenseManager.TElSBLicenseManager();
m.LicenseKey = "8F1317CD48AC4E68BABA5E339D8B365414D7ADA0289CA037E9074D29AD95FF3EC5D796BEFF0FBADB3BD82F48644C9EB810D9B5A305E0D2A1885C874D8BF974B9608CE918113FBE2AA5EEF8264C93B25ABEA98715DB4AD265F47CE02FC9952D69F2C3530B6ABAAA4C43B45E7EF6A8A0646DA038E34FBFB629C2BF0E83C6B348726E622EBD52CA05CF74C68F1279849CCD0C13EA673916BA42684015D658B8E7626F15BD826A4340EDB36CE55791A051FDBCF9FA1456C3B5008AD9990A0185C0EA3B19F9938CB7DA1FE82736ED4C7A566D4BFD53411E8380F4B020CB50E762520EFAE190836FD253B00DB18D4A485C7DC918AA4DCEC856331DD231CC8DC9C741C3";
using (var unit = GetUnitOfWork())
{
foreach (Connector connector in base.Connectors.Where(c => ((ConnectorType)c.ConnectorType).Has(ConnectorType.WebAssortment)))
{
#if DEBUG
if (connector.ConnectorID != 6)
{
continue;
}
#endif
string singleProduct = string.Empty;
try
{
log.DebugFormat("Start Magento Image Synchronization for {0}", connector.Name);
if (connector.ConnectorSystemType == null)
{
log.AuditError(string.Format("No Connector System Settings found for {0}, Magento Export can not be executed!", connector.Name), "Magento Export");
continue;
}
var type = GetType().ToString();
string serializationPath = @"C:\Magento";//default
var config = GetConfiguration();
if (config.AppSettings.Settings["AssortmentSerializationPath"] != null)
{
serializationPath = config.AppSettings.Settings["AssortmentSerializationPath"].Value;
}
var connectorSerializationPath = serializationPath;
if (!Directory.Exists(connectorSerializationPath))
{
Directory.CreateDirectory(connectorSerializationPath);
}
//#if DEBUG
// connector.Connection = "server=127.0.0.1;User Id=root;password=Om3Aih7aohQu9uPeahP4Ul3p;database=coolcat;Connect Timeout=30000;Default Command Timeout=30000;port=6014";
//#endif
ImageExporter imageExporter = new ImageExporter(connector, log, GetConfiguration(), connectorSerializationPath);
imageExporter.Execute();
}
catch (Exception ex)
{
log.Error("Error in Magento Plugin", ex);
}
var triggerIndex = connector.ConnectorSettings.GetValueByKey <bool>("ImageTriggerIndexing", false);
if (triggerIndex)
{
log.Info("Will place trigger for indexing");
try
{
var info = SftpHelper.GetFtpTriggerIndexInfo(connector);
IndexerHelper hlp = new IndexerHelper(info, "cache");
hlp.CreateImageTrigger();
log.Info("Placed a trigger file");
}
catch (Exception e)
{
log.AuditError("Couldnt upload a trigger file", e, "Magento export");
}
}
log.DebugFormat("Finished Magento Image Synchronization For {0}", connector.Name);
}
}
}
protected override void Process()
{
SBLicenseManager.TElSBLicenseManager m = new SBLicenseManager.TElSBLicenseManager();
m.LicenseKey = "8F1317CD48AC4E68BABA5E339D8B365414D7ADA0289CA037E9074D29AD95FF3EC5D796BEFF0FBADB3BD82F48644C9EB810D9B5A305E0D2A1885C874D8BF974B9608CE918113FBE2AA5EEF8264C93B25ABEA98715DB4AD265F47CE02FC9952D69F2C3530B6ABAAA4C43B45E7EF6A8A0646DA038E34FBFB629C2BF0E83C6B348726E622EBD52CA05CF74C68F1279849CCD0C13EA673916BA42684015D658B8E7626F15BD826A4340EDB36CE55791A051FDBCF9FA1456C3B5008AD9990A0185C0EA3B19F9938CB7DA1FE82736ED4C7A566D4BFD53411E8380F4B020CB50E762520EFAE190836FD253B00DB18D4A485C7DC918AA4DCEC856331DD231CC8DC9C741C3";
var connectorOverridesSetting = GetConfiguration().AppSettings.Settings["ConnectorOverrides"];
List <int> connectorOverrides = null;
if (connectorOverridesSetting != null)
{
connectorOverrides = (from p in connectorOverridesSetting.Value.Split(',') select int.Parse(p)).ToList();
}
_monitoring.Notify(Name, 0);
using (var unit = GetUnitOfWork())
{
foreach (Connector connector in base.Connectors.Where(c => ((ConnectorType)c.ConnectorType).Has(ConnectorType.WebAssortment)))
{
if (connectorOverrides != null && !connectorOverrides.Contains(connector.ConnectorID))
{
continue;
}
#if DEBUG
if (connector.ConnectorID != 5)
{
continue;
}
#endif
_monitoring.Notify(Name, connector.ConnectorID);
#if !DEBUG
SetSoldenPeriod(connector);
#endif
string singleProduct = string.Empty;
try
{
log.DebugFormat("Start Magento Assortment Export for {0}", connector.Name);
if (connector.ConnectorSystemType == null)
{
log.AuditError(string.Format("No Connector System Settings found for {}0, Magento Export can not be executed!", connector.Name), "Magento Export");
continue;
}
DateTime start = DateTime.Now;
string serializationPath = @"C:\Magento";//default
var config = GetConfiguration();
if (config.AppSettings.Settings["AssortmentSerializationPath"] != null)
{
serializationPath = config.AppSettings.Settings["AssortmentSerializationPath"].Value;
}
var connectorSerializationPath = serializationPath;
if (!Directory.Exists(connectorSerializationPath))
{
Directory.CreateDirectory(connectorSerializationPath);
}
#if DEBUG
connector.Connection = "server=127.0.0.1;User Id=root;password=Phoh9ooLaing3FieZahb7if8;database=coolcat;Connect Timeout=30000;Default Command Timeout=30000;port=5014";
#endif
MagentoExporter exporter = new MagentoExporter(connector, log, connectorSerializationPath);
exporter.Execute();
CustomerExporter cExporter = new CustomerExporter(connector, log, GetConfiguration());
cExporter.Execute();
log.DebugFormat("Finished Magento Export For {0}", connector.Name);
}
catch (OutOfMemoryException ex)
{
log.AuditCritical("Magento export -> Out of memory exception for " + connector.ConnectorID, ex, "Magento export");
_monitoring.Notify(Name, -1);
}
catch (Exception ex)
{
log.AuditError("Error in Magento Plugin", ex);
_monitoring.Notify(Name, -2);
}
var triggerIndex = connector.ConnectorSettings.GetValueByKey <bool>("TriggerIndexing", false);
if (triggerIndex)
{
log.Info("Will place trigger for indexing");
try
{
var info = SftpHelper.GetFtpTriggerIndexInfo(connector);
IndexerHelper hlp = new IndexerHelper(info, "cache");
hlp.CreateAssortmentTrigger();
log.Info("Placed a trigger file");
}
catch (Exception e)
{
log.AuditError("Couldnt upload a trigger file", e, "Magento export");
_monitoring.Notify(Name, -3);
}
}
}
_monitoring.Notify(Name, 1);
}
}
private void DownloadMessageFiles(int vendorID)
{
var remoteFileList = new ArrayList();
_downloadedFiles = new Dictionary <string, decimal>();
var ftpClient = SftpHelper.CreateClient(VendorSettingsHelper.GetSFTPSetting(vendorID), CommunicatorHelper.GetWehkampPrivateKeyFilename(vendorID), "D1r@ct379");
if (ftpClient == null)
{
log.AuditCritical("SFTP failed to connect");
return;
}
remoteFileList.Clear();
var allianceDirectoryName = string.Format("TO_{0}", VendorSettingsHelper.GetAlliantieName(vendorID).ToUpperInvariant());
try
{
ftpClient.ListDirectory(allianceDirectoryName, remoteFileList);
foreach (TElSftpFileInfo fileinfo in remoteFileList)
{
//log.Info(string.Format("Checking remote item '{0}'", fileinfo.Name));
if (fileinfo.Attributes.FileType != TSBSftpFileType.ftFile && (fileinfo.Attributes.FileType != TSBSftpFileType.ftUnknown && !fileinfo.Name.Contains(".xml")))
{
//log.Info(string.Format("Not a file. Skip item '{0}'", fileinfo.Name));
continue;
}
var message = MessageHelper.GetMessageByFilename(fileinfo.Name);
// if this is a second attempt, only try again if the validation has failed or there was an error downloading
if (message != null && message.Status != WehkampMessageStatus.ErrorDownload &&
message.Status != WehkampMessageStatus.ValidationFailed)
{
//log.Info(string.Format("Skip downloading file '{0}'", fileinfo.Name));
continue;
}
try
{
//log.Info(string.Format("Start downloading file '{0}'", fileinfo.Name));
ftpClient.DownloadFile(Path.Combine(allianceDirectoryName, fileinfo.Name).ToUnixPath(), Path.Combine(ConfigurationHelper.IncomingFilesRootFolder, vendorID.ToString(CultureInfo.InvariantCulture), fileinfo.Name));
ftpClient.RemoveFile(Path.Combine(allianceDirectoryName, fileinfo.Name).ToUnixPath());
}
catch (Exception e)
{
log.AuditError(string.Format("Cannot access {0} over SFTP: {1}\n{2}", allianceDirectoryName, e.Message, e.StackTrace));
if (message != null)
{
MessageHelper.UpdateMessageStatus(message.MessageID, WehkampMessageStatus.ErrorDownload);
}
}
finally
{
if (message != null)
{
MessageHelper.UpdateMessageAttempt(message.MessageID);
if (message.Attempts + 1 >= 10)
{
MessageHelper.UpdateMessageStatus(message.MessageID, WehkampMessageStatus.MaxRetryExceeded);
}
}
}
if (message == null || message.Attempts + 1 < 10)
{
_downloadedFiles.Add(fileinfo.Name, message != null ? message.MessageID : -1);
}
}
}
catch (Exception e)
{
log.AuditError(string.Format("Cannot access {0} over SFTP: {1}\n{2}", allianceDirectoryName, e.Message, e.StackTrace));
}
CreateMessagesFromDownloadedFiles(vendorID);
ftpClient.Dispose();
}
private void UploadMessageFiles(int vendorID)
{
var pluginLocation = GetPluginLocation();
var typesXsdPath = Path.Combine(pluginLocation, "XSD", "wdpTypes.xsd");
// first move all files to the outgoing directory
var directories = MessageHelper.GetMessageFolders("To", vendorID);
foreach (var directory in directories)
{
foreach (var file in Directory.EnumerateFiles(directory))
{
var filename = Path.GetFileName(file);
if (filename == null)
{
continue;
}
var wkm = MessageHelper.GetMessageByFilename(filename);
if (wkm != null && (wkm.MessageType == MessageHelper.WehkampMessageType.ProductAttribute || wkm.MessageType == MessageHelper.WehkampMessageType.ShipmentNotification) &&
wkm.LastModified.HasValue &&
wkm.LastModified.Value.AddMinutes(20) > DateTime.Now.ToUniversalTime())
{
continue;
}
//Only export files with the status Success
if (wkm == null || wkm.Status != WehkampMessageStatus.Success)
{
continue;
}
File.Move(Path.Combine(directory, filename), Path.Combine(ConfigurationHelper.OutgoingFilesRootFolder, vendorID.ToString(CultureInfo.InvariantCulture), filename));
}
}
var ftpClient = SftpHelper.CreateClient(VendorSettingsHelper.GetSFTPSetting(vendorID), CommunicatorHelper.GetWehkampPrivateKeyFilename(vendorID), "D1r@ct379");
if (ftpClient == null)
{
log.AuditCritical("SFTP failed to connect");
return;
}
// then we upload all of them
var remoteFileList = new ArrayList();
if (ConfigurationHelper.ListFTPFilesCheck)
{
ftpClient.ListDirectory("TO_WK", remoteFileList);
}
var remoteFiles = (TElSftpFileInfo[])remoteFileList.ToArray(typeof(TElSftpFileInfo));
foreach (var file in Directory.EnumerateFiles(Path.Combine(ConfigurationHelper.OutgoingFilesRootFolder, vendorID.ToString(CultureInfo.InvariantCulture))))
{
var filename = Path.GetFileName(file);
var message = MessageHelper.GetMessageByFilename(filename);
if (message == null || (message.Status != WehkampMessageStatus.ErrorUpload && message.Status != WehkampMessageStatus.Success))
{
continue;
}
if (message.Status == WehkampMessageStatus.Success)
{
MessageHelper.ResetMessageAttempt(message.MessageID);
}
var error = false;
MessageHelper.UpdateMessageAttempt(message.MessageID);
if (!ConfigurationHelper.ListFTPFilesCheck || remoteFiles.All(c => c.Name != file))
{
try
{
var path = Path.Combine(ConfigurationHelper.OutgoingFilesRootFolder, message.VendorID.ToString(CultureInfo.InvariantCulture), filename);
var doc = XDocument.Load(path);
if (doc.Root == null)
{
continue;
}
if (_downloadedFiles.ContainsKey(filename))
{
_downloadedFiles[filename] = message.MessageID;
}
else
{
_downloadedFiles.Add(filename, message.MessageID);
}
MessageHelper.UpdateMessageRecieved(message.MessageID, DateTime.Now.ToUniversalTime());
var xsdPath = Path.Combine(pluginLocation, "XSD", string.Format("{0}.xsd", doc.Root.Name.LocalName));
if (!File.Exists(xsdPath) || !File.Exists(typesXsdPath))
{
throw new FileNotFoundException(String.Format("Cannot find Xsd validation files\n{0}\n{1}", xsdPath, typesXsdPath));
}
var schemas = new XmlSchemaSet();
schemas.Add(null, XElement.Load(xsdPath).CreateReader());
schemas.Add(null, XElement.Load(typesXsdPath).CreateReader());
schemas.Compile();
doc.Validate(schemas, null);
ftpClient.UploadFile(path, Path.Combine("TO_WK", filename).ToUnixPath(), TSBFileTransferMode.ftmOverwrite);
File.Move(Path.Combine(ConfigurationHelper.OutgoingFilesRootFolder, message.VendorID.ToString(CultureInfo.InvariantCulture), filename), Path.Combine(ConfigurationHelper.ArchivedRootFolder, message.VendorID.ToString(CultureInfo.InvariantCulture), filename));
MessageHelper.UpdateMessageStatus(message.MessageID, WehkampMessageStatus.Archived);
MessageHelper.UpdateMessagePath(message.MessageID, ConfigurationHelper.ArchivedRootFolder);
MessageHelper.UpdateMessageSent(message.MessageID, DateTime.Now.ToUniversalTime());
}
catch (Exception e)
{
if (e is XmlSchemaValidationException)
{
MessageHelper.UpdateMessageStatus(_downloadedFiles[filename], WehkampMessageStatus.ValidationFailed);
log.AuditError(string.Format("XML validation failed for file {0} : {1}\n{2}", filename, e.Message, e.StackTrace));
}
else
{
MessageHelper.UpdateMessageStatus(message.MessageID, WehkampMessageStatus.ErrorUpload);
log.AuditError(string.Format("Cannot upload file {0}", filename), e);
}
MessageHelper.UpdateMessagePath(message.MessageID, ConfigurationHelper.OutgoingFilesRootFolder);
error = true;
}
}
else
{
MessageHelper.UpdateMessageStatus(message.MessageID, WehkampMessageStatus.ErrorUpload);
MessageHelper.UpdateMessagePath(message.MessageID, ConfigurationHelper.OutgoingFilesRootFolder);
log.AuditError(string.Format("Cannot upload file {0}, File already exists on remote server.", filename));
error = true;
}
if (error && message.Attempts + 1 > 10)
{
MessageHelper.UpdateMessageStatus(message.MessageID, WehkampMessageStatus.MaxRetryExceeded);
File.Move(Path.Combine(ConfigurationHelper.OutgoingFilesRootFolder, filename), Path.Combine(ConfigurationHelper.FailedFilesRootFolder, filename));
MessageHelper.UpdateMessagePath(message.MessageID, ConfigurationHelper.FailedFilesRootFolder);
}
}
ftpClient.Dispose();
}
static void RunProgram()
{
string jobId = DateTime.Now.ToString("yyyyMMddhhmmss");
try
{
Log.Debug("Processing in progress...");
// Basic validation for sftp config
if (string.IsNullOrEmpty(AppSettings.SftpHost) ||
string.IsNullOrEmpty(AppSettings.SftpUsername) ||
string.IsNullOrEmpty(AppSettings.SftpPassword) ||
string.IsNullOrEmpty(AppSettings.SftpDownloadDirectory) ||
string.IsNullOrEmpty(AppSettings.SftpDownloadFilename) ||
string.IsNullOrEmpty(AppSettings.SftpUploadDirectory))
{
Log.Error("Missing SFTP Configuration.");
SendSystemEmail(Constants.StatusEmailSubject + " - Configuration Error", "Missing SFTP Configuration.");
return;
}
if (!SftpHelper.VerifyConnection(out string error))
{
Log.Error("Unable to connect to SFTP, " + error);
SendSystemEmail(Constants.StatusEmailSubject + " - Failed to connect to SFTP", "Please ensure the SFTP Configuration is valid.<h3>Exception</h3>" + error);
return;
}
string downloadFolderPath = Path.Combine(AppSettings.DownloadPath, jobId);
string workingFolderPath = Path.Combine(AppSettings.WorkingPath, jobId);
var parser = new DataParser();
parser.LoadParserSettings();
var allData = new List <ParsedData>();
var failedRows = new List <Tuple <int, string, string> >();
// Download from sftp remote server
DownloadedFile downloadedFile = SftpHelper.DownloadFile(AppSettings.SftpDownloadDirectory, AppSettings.SftpDownloadFilename, downloadFolderPath);
// Terminate if no files available for download
if (downloadedFile == null)
{
Log.Error("No files available for download");
SendSystemEmail(Constants.StatusEmailSubject, "No files available for download.");
FileHelper.DeleteFile(downloadFolderPath);
return;
}
//Read from downloaded csv and parse into order list
var rows = CsvUtil.GetRecords <InOrderConfirmation>(downloadedFile.FullName);
var rowIndex = 1; //csv header consider as 1 row
foreach (var row in rows)
{
rowIndex++;
var data = parser.ParseXML(row.OrderXML, out List <string> errors, enableThrowException: false);
if (data != null && !errors.Any())
{
allData.AddRange(data);
}
else
{
failedRows.Add(new Tuple <int, string, string>(rowIndex, row.OrderNumber, string.Join(",", errors)));
}
}
// Terminate if any invalid rows
if (failedRows.Any())
{
Log.Error("Invalid Records Found");
SendParseErrorEmail(jobId, failedRows);
ArchiveFiles(jobId, true);
return;
}
// Process Export and Upload
var results = new Dictionary <string, UploadResult>();
if (allData.Any())
{
FileHelper.CreateDirectory(workingFolderPath);
// Flatten the data
var flattenData = allData.GroupBy(x => x.Name)
.Select(group => new ParsedData
{
Name = group.Key,
Data = group.SelectMany(item => item.Data).Distinct().ToList()
}).ToList();
// Export csv and upload to SFTP
foreach (var item in flattenData)
{
string filename = item.Name + "_" + jobId + ".csv";
string path = workingFolderPath + " /" + filename;
try
{
parser.ExportAsCsv(item.Data, path);
var result = UploadExportCsvToSftp(AppSettings.SftpUploadDirectory, path);
results.Add(filename, result);
}
catch (Exception ex)
{
results.Add(filename, new UploadResult
{
Success = false,
ErrorMessage = "Failed to export csv for upload. Exception: " + ex.ToString()
});
}
}
}
// Send status result email
SendStatusEmail(jobId, results);
}
catch (Exception ex)
{
Log.Error("RunProgram Error", ex);
SendSystemEmail(Constants.StatusEmailSubject + " - Task Error", ex.ToString());
}
finally
{
ArchiveFiles(jobId, true);
}
}
private bool ProcessProductImages(Database db, List <ProductMedia> productMedias, int vendorID)
{
if (!productMedias.Any())
{
return(true);
}
var attributeid = db.FirstOrDefault <int>("SELECT AttributeID FROM ProductAttributeMetaData WHERE AttributeCode = @0", "WehkampProductNumber");
if (attributeid == 0)
{
throw new ConfigurationErrorsException("Attribute \"WehkampProductNumber\" is missing from the ProductAttributeMetadata table");
}
var wehkampPids = new Dictionary <int, string>();
if (productMedias.Count <= 500)
{
var productMediaArray = string.Join(",", productMedias.Select(c => c.ProductID).ToArray());
wehkampPids = db.Dictionary <int, string>(string.Format("SELECT ProductID, Value FROM ProductAttributeValue WHERE AttributeID = {1} AND ProductID IN ({0})", productMediaArray, attributeid));
}
else
{
for (var i = 0; i < productMedias.Count; i += 500)
{
var productMediaArray = string.Join(",", productMedias.Skip(i).Take(Math.Min(500, productMedias.Count - i)).Select(c => c.ProductID).ToArray());
var temp = db.Dictionary <int, string>(string.Format("SELECT ProductID, Value FROM ProductAttributeValue WHERE AttributeID = {1} AND ProductID IN ({0})", productMediaArray, attributeid));
foreach (var pm in temp)
{
if (!wehkampPids.ContainsKey(pm.Key))
{
wehkampPids.Add(pm.Key, pm.Value);
}
}
}
}
var ftpClient = SftpHelper.CreateClient(VendorSettingsHelper.GetSFTPSetting(vendorID), CommunicatorHelper.GetWehkampPrivateKeyFilename(vendorID), "D1r@ct379");
if (ftpClient == null)
{
log.AuditCritical("SFTP failed to connect");
return(false);
}
string manid, path, newFilename;
int index;
foreach (var media in productMedias)
{
if (!wehkampPids.ContainsKey(media.ProductID))
{
continue;
}
manid = media.ManufacturerID.Split(' ')[0];
if (string.IsNullOrEmpty(manid))
{
continue;
}
index = media.ImagePath.LastIndexOf(manid, StringComparison.InvariantCultureIgnoreCase);
if (index < 0)
{
continue;
}
path = Path.Combine(ConfigurationHelper.FTPMediaDirectory, media.ImagePath);
if (!File.Exists(path))
{
log.Error(string.Format("Could not find image file {0}", path));
continue;
}
newFilename = string.Format("{0}_{1}", wehkampPids[media.ProductID], media.ImagePath.Substring(index));
try
{
//Only upload files ending on:
//1. _b.png - Back
//2. _f.png - Front
//3. _h.png - Hoover
//if (newFilename.ToLowerInvariant().EndsWith("_b.png") || newFilename.ToLowerInvariant().EndsWith("_f.png") || newFilename.ToLowerInvariant().EndsWith("_h.png"))
if (newFilename.ToLowerInvariant().EndsWith("_b.png") || newFilename.ToLowerInvariant().EndsWith("_f.png") || newFilename.ToLowerInvariant().EndsWith("_h.png"))
{
ftpClient.UploadFile(path, Path.Combine("TO_WK", "Pictures", newFilename).ToUnixPath(), TSBFileTransferMode.ftmOverwrite);
}
}
catch (Exception e)
{
log.Error(string.Format("Cannot upload file : {0}", media.ImagePath), e);
return(false);
}
}
return(true);
}
