private async Task ProcessImportFileQuery(IQueryable <ImportedFile> query)
{
// order is important for file processing
var importFiles = query.OrderByDescending(f => f.OriginalFileName.StartsWith(_fileNameMatcher.AcctStatusReportNonEcaSqlLike))
.ThenByDescending(f => f.OriginalFileName.StartsWith(_fileNameMatcher.AcctStatusReportSqlLike))
.ThenByDescending(f => f.OriginalFileName.Contains(_fileNameMatcher.SytossClientInfoSqlLike))
.ThenByDescending(f => f.FileCreateDate)
.ToList();
if (!importFiles.Any())
{
GlobalLogger.LogInfo("Job has not found new files for importing. Waiting...", GetType().Name, true);
}
else
{
foreach (var file in importFiles)
{
try
{
await TryImportFile(file);
}
catch (Exception ex)
{
file.FileState = FileState.Imported;
file.FileStatus = FileStatus.Failed;
_importJobRepository.UpdateImportedFile(file);
_importJobRepository.SaveChanges();
GlobalLogger.LogError($"Error while importing file {file?.OriginalFileName}.", ex, GetType().Name, true);
}
}
}
}
protected override void OnUpdate(long now)
{
if (_fireRunner != null)
{
try {
_fireRunner.UpdateNow(now);
} catch (Exception e) {
GlobalLogger.LogError(e);
_status = CompStatus.COMPLETE;
}
}
}
protected override void OnEnter()
{
_status = CompStatus.COMPLETE;
LuaTable tb = machine.GetRuntimeData <LuaTable>(CpxState.TEMP_DATA);
if (tb == null)
{
GlobalLogger.LogError("no setting data retrieved");
return;
}
Vector3 pos;
if (!tb.TryGet("position", out pos))
{
pos = model.position;
}
string prefab;
if (!tb.TryGet("prefab", out prefab))
{
GlobalLogger.LogError(string.Format("{0} require prefab", this));
return;
}
string text;
if (!tb.TryGet("text", out text))
{
GlobalLogger.LogError(string.Format("{0} require text", this));
return;
}
string body;
if (!tb.TryGet("body", out body))
{
body = "body";
}
string bone;
if (!tb.TryGet("head", out bone))
{
bone = RoleBone.HEAD;
}
Transform me = model.FindBone(body, bone);
Vector3 src = me ? me.position : model.position;
FlyTextEntity fly = new FlyTextEntity {
@from = src,
to = (src - pos) + src,
prefab = prefab,
text = text
};
fly.Initialize();
}
protected override void OnEnter()
{
_status = CompStatus.COMPLETE;
object obj = machine.GetRuntimeData <object>(CpxState.TEMP_DATA);
if (obj == null)
{
GlobalLogger.LogError("no setting data retrieved");
return;
}
_fighter = entity as IFightableEntity;
if (_fighter == null)
{
return;
}
_fx = GetFireFx(obj);
if (_fx == null)
{
return;
}
LuaTable tb = obj as LuaTable;
Vector3 pos;
if (tb != null && tb.TryGet("position", out pos))
{
model.MoveTo(pos);
}
float orient;
if (tb != null && tb.TryGet("orient", out orient))
{
Debug.Log(string.Format("model.orient ={0}, data.orient ={1}", model.orientation, orient));
model.orientation = orient;
}
_status = CompStatus.RUNING;
_fireRunner = new FxFireRunner(_fighter, _fx)
{
targetInfos = new FireFxInfo[0],
targetPos = GetTargetPosition(tb)
};
_fireRunner.completeEvent.AddListener(OnComplete);
try {
_fireRunner.Start();
} catch (Exception e) {
GlobalLogger.LogError(e);
_status = CompStatus.COMPLETE;
}
}
private IEnumerator Create()
{
string file;
if (!AssetUtils.ParseFile(prefab, out file))
{
yield break;
}
IEnumerator e = AssetUtils.Load(AssetType.Prefab, file);
while (e.MoveNext())
{
yield return(null);
}
e = AssetUtils.PickAsyn <GameObject>(AssetType.Prefab, file);
while (e.MoveNext())
{
yield return(null);
}
GameObject p = AssetUtils.Pick <GameObject>(AssetType.Prefab, file);
if (!p)
{
GlobalLogger.LogError(string.Format("prefab read error at {0}", file));
yield break;
}
_model = GameObjectPool.Get(p, GameObjectPool.LifeMode.SceneLife);
Transform t = _model.transform;
t.SetParent(CanvasManager.sceneCanvas);
t.position = from;
_model.SetActive(true);
Transform child = t.GetChild(0);
float orient = VectorMath.CalcOrientation(to - from);
t.eulerAngles = new Vector3(0, orient, 0);
Text3D_Dll t3 = child.GetComponent <Text3D_Dll>();
if (t3)
{
t3.text = text;
e = t3.DoAnimation();
while (e.MoveNext())
{
yield return(null);
}
}
}
public static FireFxRoot GetFireSetting(string path)
{
if (File.Exists(path))
{
TreeEditorFactory.Start();
XDocument doc = XDocument.Load(path);
XElement root = doc.Root;
string canvasType = StringDataTools.XAttribute(root, "type");
TreeCanvas canvas = TreeEditorFactory.GetCanvas(canvasType);
canvas.Read(root);
FireFxRoot r = new FireFxRoot();
new TreeToRuntimeComp(canvas, TreeFactory.CreateInstance, r).Transfer();
return(r);
}
GlobalLogger.LogError(string.Format("{0} is not a valid path", path));
return(null);
}
protected override void OnEnter()
{
_arrive = false;
long now = SystemTime.GetTime();
long extra = (long)(cpx.duration * 1000);
_duetime = now + extra;
model.SetAnimator(AnimatorType.IS_RUNNING, AnimatorControllerParameterType.Bool, 1);
_steer = model.GetAttach <Steer>() ?? model.Attach(new Steer());
_start = model.position;
_calcPos = _start;
_position = _start;
LuaTable tb = machine.GetRuntimeData <LuaTable>(CpxState.TEMP_DATA);
Vector3 dest, pos;
if (!tb.TryGet("position", out pos))
{
GlobalLogger.LogError(string.Format("{0} require position", this));
_status = CompStatus.COMPLETE;
return;
}
if (!tb.TryGet("forcast", out dest))
{
GlobalLogger.LogError(string.Format("{0} require forcast", this));
_status = CompStatus.COMPLETE;
return;
}
_isLast = (bool)tb["isLast"];
if (now - _steer.prevTime > extra)
{
_steer.prevTime = now;
}
_prevTime = _steer.prevTime;
if (_isLast)
{
_destiny = pos;
}
else
{
Vector3 dir = dest - pos;
VectorMath.Truncate(ref dir, model.speed * 0.1f);
_destiny = dir + pos;
}
_status = CompStatus.RUNING;
}
protected override void OnEnter()
{
long now = SystemTime.GetTime();
model.SetAnimator(AnimatorType.IS_RUNNING, AnimatorControllerParameterType.Bool, 1);
LuaTable tb = machine.GetRuntimeData <LuaTable>(CpxState.TEMP_DATA);
Vector3 np;
if (!tb.TryGet("position", out np))
{
GlobalLogger.LogError(string.Format("{0} require position", this));
_status = CompStatus.COMPLETE;
return;
}
Vector3 dest;
if (!tb.TryGet("forcast", out dest))
{
GlobalLogger.LogError(string.Format("{0} require forcast", this));
_status = CompStatus.COMPLETE;
return;
}
Vector3 mp = model.position;
Vector3 gap = np - mp;
float offset = gap.magnitude;
if (offset > cpx.threshold)
{
model.MoveTo(np);
mp = np;
}
_path[0] = mp;
_path[1] = np;
_path[2] = dest;
_runner = MoveRunner_Path3D.Get();
_runner.Set(model, this);
_runner.Preset(_path, now, 0);
_runner.updateAnim = false;
_runner.Enter();
_status = CompStatus.RUNING;
}
protected override void OnEnter()
{
_status = CompStatus.COMPLETE;
LuaTable tb = machine.GetRuntimeData <LuaTable>(CpxState.TEMP_DATA);
if (tb == null)
{
GlobalLogger.LogError("no setting data retrieved");
return;
}
string prefab;
if (!tb.TryGet("prefab", out prefab))
{
GlobalLogger.LogError(string.Format("{0} need prefab", this));
return;
}
double endTime;
if (!tb.TryGet("duetime", out endTime))
{
GlobalLogger.LogError(string.Format("{0} need duetime", this));
return;
}
string bone;
if (!tb.TryGet("bone", out bone))
{
bone = string.Empty;
}
string body;
if (!tb.TryGet("body", out body))
{
body = "body";
}
BuffRenderer renderer = new BuffRenderer(prefab, body, bone, (long)endTime);
model.Attach(renderer);
}
private async Task TryLoadFileAsync(ImportedFile file)
{
try
{
file.FileState = FileState.Loaded;
file.FileStatus = FileStatus.Success;
file.CopiedDate = DateTime.UtcNow;
await _systemManager.SaveFileAsync(file);
GlobalLogger.LogInfo($"File copied: {file.OriginalFileName} from ftp: [{file.FtpCredential.FtpName}].", GetType().Name, true);
}
catch (Exception ex)
{
file.FileStatus = FileStatus.Failed;
GlobalLogger.LogError($"Error while copying file: {file.OriginalFileName} from ftp: [{file.FtpCredential.FtpName}].", ex, GetType().Name, true);
}
finally
{
_copyJobRepository.UpdateImportedFile(file);
_copyJobRepository.SaveChanges();
}
}
protected override void OnEnter()
{
model.SetAnimator(AnimatorType.IS_RUNNING, AnimatorControllerParameterType.Bool, 0);
LuaTable tb = machine.GetRuntimeData <LuaTable>(CpxState.TEMP_DATA);
bool isLast = (bool)tb["isLast"];
if (isLast)
{
if (!tb.TryGet("forcast", out _destiny))
{
GlobalLogger.LogError(string.Format("{0} require forcast", this));
_status = CompStatus.COMPLETE;
return;
}
_position = model.position;
_status = CompStatus.RUNING;
}
else
{
_status = CompStatus.COMPLETE;
}
_prevTime = SystemTime.GetTime();
}
protected override void OnEnter()
{
long now = SystemTime.GetTime();
long extra = (long)(cpx.duration * 1000.0f);
_duetime = now + extra;
model.SetAnimator(AnimatorType.IS_RUNNING, AnimatorControllerParameterType.Bool, 1);
LuaTable tb = machine.GetRuntimeData <LuaTable>(CpxState.TEMP_DATA);
Vector3 dest;
if (!tb.TryGet("forcast", out dest))
{
GlobalLogger.LogError(string.Format("{0} require forcast", this));
_status = CompStatus.COMPLETE;
return;
}
Vector3 pos;
if (!tb.TryGet("position", out pos))
{
GlobalLogger.LogError(string.Format("{0} require position", this));
_status = CompStatus.COMPLETE;
return;
}
Vector3 dir = dest - pos;
VectorMath.Truncate(ref dir, model.speed * 0.1f);
_destiny = dir + pos;
_position = model.position;
dir = _position - pos;
if (dir.magnitude > cpx.updateThreshold)
{
_position = pos;
model.MoveTo(_position);
}
_status = CompStatus.RUNING;
}
private async Task <FileStatus> ProcessFeesFileReport(ImportedFile file, Stream stream)
{
if (stream == null || stream.Length == 0)
{
return(await Task.FromResult(FileStatus.Failed));
}
var newTradeFeeList = new List <TradeFee>();
using (var reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
var row = await reader.ReadLineAsync();
if (!row.Trim().StartsWith("<ClientFee "))
{
continue;
}
try
{
var doc = XDocument.Load(new MemoryStream(Encoding.UTF8.GetBytes(row)));
var e = doc.Elements().First();
var tradeFeeTypeId = GetOrCreateTradeFeeTypeId(new TradeFeeType
{
TradeFeeTypeName = e.Attribute(XName.Get("feeType"))?.Value
});
var tradeFeeInstrumentId = GetOrCreateTradeFeeInstrumentId(new TradeInstrument
{
InstrumentName = e.Attribute(XName.Get("currency"))?.Value
});
var tradeAccountId = GetOrCreateTradeAccountId(new TradeAccount
{
AccountName =
_fileNameMatcher.GetCorrectAccountId(e.Attribute(XName.Get("accountId"))?.Value),
AccountAlias = e.Attribute(XName.Get("acctAlias"))?.Value,
MasterAccountId = file.MasterAccountId,
ImportedFile = file
});
var externalDate = DateHelper.ParseDateTime(e.Attribute(XName.Get("date"))?.Value,
"yyyyMMdd;HHmmss", DateHelper.DefaultDateFormat);
if (externalDate == null)
{
throw new Exception("There is no date");
}
var newTradeFee = new TradeFee
{
TradeAccountId = tradeAccountId,
TradeFeeTypeId = tradeFeeTypeId,
TradeInstrumentId = tradeFeeInstrumentId,
ImportedFile = file,
ExternalTradeName = e.Attribute(XName.Get("tradeID"))?.Value,
ExternalExecName = e.Attribute(XName.Get("execID"))?.Value,
ExternalDate = externalDate.Value,
RateToBase = ParseDecimal(e.Attribute(XName.Get("fxRateToBase"))?.Value),
RevenueInCurrency = ParseDecimal(e.Attribute(XName.Get("revenue"))?.Value),
RevenueInBase = ParseDecimal(e.Attribute(XName.Get("revenueInBase"))?.Value),
ExpenseInCurrency = ParseDecimal(e.Attribute(XName.Get("expense"))?.Value),
ExpenseInBase = ParseDecimal(e.Attribute(XName.Get("expenseInBase"))?.Value),
NetInCurrency = ParseDecimal(e.Attribute(XName.Get("net"))?.Value),
NetInBase = ParseDecimal(e.Attribute(XName.Get("netInBase"))?.Value),
};
newTradeFeeList.Add(newTradeFee);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while parsing TradeFee record. \nRow: {row}", ex, GetType().Name,
true);
}
}
}
try
{
_importJobRepository.AddRangeTradeFee(newTradeFeeList);
_importJobRepository.SaveChanges();
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while adding TradeFee records to DB.", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
return(await Task.FromResult(FileStatus.Success));
}
private async Task <FileStatus> ProcessAcctStatusReportFileReport(ImportedFile file, string nameBaseNode,
string sqlLikeExpression, Stream stream)
{
if (stream == null || stream.Length == 0)
{
return(await Task.FromResult(FileStatus.Failed));
}
var tradeAccountsForAdding = new List <TradeAccount>();
var tradeAccountsForUpdating = new List <TradeAccount>();
if (stream.Position > 0)
{
stream.Position = 0;
}
var document = XDocument.Load(stream);
var elements = document.Element(nameBaseNode)?.Element("Accounts")?.Elements("Account").ToList();
if (elements != null && elements.Any())
{
elements.ForEach(e =>
{
try
{
var tradeAccount = new TradeAccount
{
AccountName = _fileNameMatcher.GetCorrectAccountId(e.Attribute(XName.Get("id"))?.Value),
MasterAccountId = file.MasterAccountId,
AccountAlias = e.Attribute(XName.Get("alias"))?.Value,
DateOpened = DateHelper.ParseDateTime(e.Attribute(XName.Get("date_opened"))?.Value),
DateClosed = DateHelper.ParseDateTime(e.Attribute(XName.Get("date_closed"))?.Value),
DateFunded = DateHelper.ParseDateTime(e.Attribute(XName.Get("date_funded"))?.Value),
TradeStatus = e.Attribute(XName.Get("status"))?.Value,
Mobile = e.Attribute(XName.Get("mobile"))?.Value
};
if (_tradeAccountModel.Contains(tradeAccount.AccountName))
{
tradeAccount.Id = _tradeAccountModel.GetById(tradeAccount.AccountName).Id;
tradeAccount.ImportedFileId = file.Id;
tradeAccountsForUpdating.Add(tradeAccount);
}
else
{
tradeAccount.ImportedFile = file;
tradeAccountsForAdding.Add(tradeAccount);
}
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while parsing TradeCash record. \nRow: {e}", ex, GetType().Name,
true);
}
});
}
try
{
_importJobRepository.AddRangeTradeAccounts(tradeAccountsForAdding);
_importJobRepository.SaveChanges();
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while adding TradeAccount records to DB.", ex, GetType().Name,
true);
return(await Task.FromResult(FileStatus.Failed));
}
foreach (var tradeAccount in tradeAccountsForUpdating)
{
try
{
_importJobRepository.AddOrUpdateTradeAccount(tradeAccount);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while updating TradeAccount record to DB.", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
}
_importJobRepository.SaveChanges();
GlobalLogger.LogInfo($"Trade Accounts: [{tradeAccountsForAdding.Count}] added.", GetType().Name, true);
GlobalLogger.LogInfo($"Trade Accounts: [{tradeAccountsForUpdating.Count}] updated.", GetType().Name, true);
RefillTradeAccounts();
GC.Collect();
return(await Task.FromResult(FileStatus.Success));
}
private async Task <FileStatus> ProcessSytossClientInfoReport(ImportedFile file, Stream stream)
{
if (stream == null || stream.Length == 0)
{
return(await Task.FromResult(FileStatus.Failed));
}
var tradeAccountsForUpdating = new List <TradeAccount>();
using (var reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
var row = await reader.ReadLineAsync();
if (!row.Trim().StartsWith("<AccountInformation"))
{
continue;
}
try
{
var doc = XDocument.Load(new MemoryStream(Encoding.UTF8.GetBytes(row)));
if (!doc.Elements().Any())
{
continue;
}
var e = doc.Elements().First();
var tradeAccId = GetOrCreateTradeAccountId(new TradeAccount
{
AccountName = _fileNameMatcher.GetCorrectAccountId(e.Attribute(XName.Get("accountId"))?.Value),
AccountAlias = e.Attribute(XName.Get("acctAlias"))?.Value,
MasterAccountId = file.MasterAccountId,
ImportedFile = file
});
var tradeAccount = _importJobRepository.GetTradeAccountById(tradeAccId);
var tradingPermsStr = e.Attribute(XName.Get("tradingPermissions"))?.Value;
ProcessTradingPermissions(tradingPermsStr, tradeAccount);
tradeAccount.Name = e.Attribute(XName.Get("name"))?.Value;
tradeAccount.City = e.Attribute(XName.Get("city"))?.Value;
tradeAccount.DateFunded = DateHelper.ParseDate(e.Attribute(XName.Get("dateFunded"))?.Value);
tradeAccount.СountryResidentialAddress = e.Attribute(XName.Get("countryResidentialAddress"))?.Value;
tradeAccount.CityResidentialAddress = e.Attribute(XName.Get("cityResidentialAddress"))?.Value;
tradeAccount.StateResidentialAddress = e.Attribute(XName.Get("stateResidentialAddress"))?.Value;
tradeAccount.IbEntity = e.Attribute(XName.Get("ibEntity"))?.Value;
tradeAccount.AccountType = e.Attribute(XName.Get("accountType"))?.Value;
tradeAccount.StreetResidentialAddress = e.Attribute(XName.Get("streetResidentialAddress"))?.Value;
tradeAccount.PostalCode = e.Attribute(XName.Get("postalCode"))?.Value;
tradeAccount.PrimaryEmail = e.Attribute(XName.Get("primaryEmail"))?.Value;
tradeAccount.Currency = e.Attribute(XName.Get("currency"))?.Value;
tradeAccount.AccountCapabilities = e.Attribute(XName.Get("accountCapabilities"))?.Value;
tradeAccount.CustomerType = e.Attribute(XName.Get("customerType"))?.Value;
tradeAccount.IsClientInfo = true;
tradeAccountsForUpdating.Add(tradeAccount);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while parsing ClientInfo record. \nRow: {row}", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
}
}
foreach (var tradeAccount in tradeAccountsForUpdating)
{
try
{
_importJobRepository.AddOrUpdateTradeAccount(tradeAccount);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while updating Client info of TradeAccount.", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
}
_importJobRepository.SaveChanges();
RefillTradeAccounts();
return(await Task.FromResult(FileStatus.Success));
}
protected override async Task RunInternal()
{
SyncerInfoService.Log(GetType().Name, JobStatus.Started.ToString());
using (_ftpJobRepository.BeginOperation())
{
var ftpCredentials = _ftpJobRepository.GetFtpCredentialList();
if (!ftpCredentials.Any())
{
GlobalLogger.LogInfo("Job has not found any ftp credentials. Waiting...", GetType().Name, true);
}
else
{
var existingFiles = _ftpJobRepository.ImportedFileList();
var existingFileNames = existingFiles.Select(ef => ef.OriginalFileName);
foreach (var cred in ftpCredentials)
{
var totalCount = 0;
List <(string fileName, string accountName, DateTime creationDate)> newFiles;
var accountNameList = cred.MasterAccounts.Select(m => m.AccountName).ToList();
try
{
newFiles = (await _ftpLoader.LoadFilesAsync(cred))
.Where(f => f.creationDate >= _fromDate && accountNameList.Contains(f.accountName) && !existingFileNames.Contains(f.fileName))
// order is important for file processing
.OrderByDescending(f => _fileNameMatcher.IsAcctStatusReportNonEca(f.fileName))
.ThenByDescending(f => _fileNameMatcher.IsAcctStatusReport(f.fileName))
.ThenByDescending(f => f.creationDate)
.ToList();
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while trying to load files from ftp: [{cred.FtpName}]", ex, GetType().Name, true);
return;
}
FilterFilesByMaxCreateDate(newFiles, existingFiles, _fileNameMatcher.AcctStatusReportNonEcaSqlLike, cred.MasterAccounts);
FilterFilesByMaxCreateDate(newFiles, existingFiles, _fileNameMatcher.AcctStatusReportSqlLike, cred.MasterAccounts);
FilterFilesByMaxCreateDate(newFiles, existingFiles, _fileNameMatcher.SytossClientInfoSqlLike, cred.MasterAccounts);
if (!newFiles.Any())
{
GlobalLogger.LogInfo($"Job has not found new files in [{cred.FtpName}]. Waiting...", GetType().Name, true);
}
else
{
GlobalLogger.LogInfo($"Job has found new: [{newFiles.Count()}] files on ftp: [{cred.FtpName}].", GetType().Name, true);
var transitFiles = _ftpJobRepository.GetTransitFileList();
foreach (var file in newFiles)
{
if (!existingFiles.All(f => f.OriginalFileName != file.fileName))
{
continue;
}
var newFile = new ImportedFile()
{
CreatedDate = DateTime.UtcNow,
ModifiedDate = DateTime.UtcNow,
OriginalFileName = file.fileName,
FileCreateDate = file.creationDate,
MasterAccount = cred.MasterAccounts.FirstOrDefault(a => a.AccountName == file.accountName),
FtpCredential = cred,
FileState = FileState.Registered,
FileStatus = FileStatus.Success,
RegisteredDate = DateTime.UtcNow
};
if (transitFiles.Any(f => f.AccountName == file.accountName && file.fileName.Contains(f.OriginalFileName)))
{
newFile.FileUpload = new FileUpload();
}
_ftpJobRepository.AddImportedFile(newFile);
totalCount++;
GlobalLogger.LogInfo($"New file: {newFile.OriginalFileName} registered from ftp: [{cred.FtpName}].", GetType().Name, true);
}
try
{
_ftpJobRepository.SaveChanges();
GlobalLogger.LogInfo($"Amount registered: [{totalCount}] files in DB from ftp: [{cred.FtpName}].", GetType().Name, true);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while trying to save registered files from ftp: [{cred.FtpName}]", ex, GetType().Name, true);
return;
}
}
}
}
}
SyncerInfoService.Log(GetType().Name, JobStatus.Stopped.ToString());
}
private async Task <FileStatus> ProcessTradesAsReport(ImportedFile file, Stream stream)
{
if (stream == null || stream.Length == 0)
{
return(await Task.FromResult(FileStatus.Failed));
}
var newTradeAsList = new List <TradeTradesAs>();
using (var reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
var row = await reader.ReadLineAsync();
if (!row.Trim().StartsWith("<AssetSummary "))
{
continue;
}
try
{
var doc = XDocument.Load(new MemoryStream(Encoding.UTF8.GetBytes(row)));
if (!doc.Elements().Any())
{
continue;
}
var e = doc.Elements().First();
var tradeAccId = GetOrCreateTradeAccountId(new TradeAccount
{
AccountName =
_fileNameMatcher.GetCorrectAccountId(e.Attribute(XName.Get("accountId"))?.Value),
AccountAlias = e.Attribute(XName.Get("acctAlias"))?.Value,
MasterAccountId = file.MasterAccountId,
ImportedFile = file
});
var newTradeAs = new TradeTradesAs
{
TradeAccountId = tradeAccId,
ReportDate = file.FileCreateDate,
Quantity = ParseDecimal(e.Attribute(XName.Get("quantity"))?.Value),
AssetCategory = e.Attribute(XName.Get("assetCategory"))?.Value,
ImportedFile = file
};
newTradeAsList.Add(newTradeAs);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while parsing TradeAs record. \nRow: {row}", ex, GetType().Name,
true);
}
}
}
try
{
_importJobRepository.AddRangeTradeAs(newTradeAsList);
_importJobRepository.SaveChanges();
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while adding TradeAs records to DB.", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
return(await Task.FromResult(FileStatus.Success));
}
private async Task <FileStatus> ProcessTradesExeReport(ImportedFile file, Stream stream)
{
if (stream == null || stream.Length == 0)
{
return(await Task.FromResult(FileStatus.Failed));
}
var newTradeExeList = new List <TradesExe>();
using (var reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
var row = await reader.ReadLineAsync();
if (!row.Trim().StartsWith("<Trade "))
{
continue;
}
try
{
var doc = XDocument.Load(new MemoryStream(Encoding.UTF8.GetBytes(row)));
if (!doc.Elements().Any())
{
continue;
}
var e = doc.Elements().First();
var tradeAccId = GetOrCreateTradeAccountId(new TradeAccount
{
AccountName =
_fileNameMatcher.GetCorrectAccountId(e.Attribute(XName.Get("accountId"))?.Value),
AccountAlias = e.Attribute(XName.Get("acctAlias"))?.Value,
MasterAccountId = file.MasterAccountId,
ImportedFile = file
});
var newTradeExe = new TradesExe
{
TradeAccountId = tradeAccId,
Quantity = ParseDecimal(e.Attribute(XName.Get("quantity"))?.Value),
TradePrice = ParseDecimal(e.Attribute(XName.Get("tradePrice"))?.Value),
ClosePrice = ParseDecimal(e.Attribute(XName.Get("closePrice"))?.Value),
VolatilityOrderLink =
long.TryParse(e.Attribute(XName.Get("volatilityOrderLink"))?.Value, out var res)
? res
: 0,
AssetCategory = e.Attribute(XName.Get("assetCategory"))?.Value,
PutCall = e.Attribute(XName.Get("putCall"))?.Value,
BuySell = e.Attribute(XName.Get("buySell"))?.Value,
Description = e.Attribute(XName.Get("description"))?.Value,
OrderType = e.Attribute(XName.Get("orderType"))?.Value,
OrderReference = e.Attribute(XName.Get("orderReference"))?.Value,
Currency = e.Attribute(XName.Get("currency"))?.Value,
IbExecId = e.Attribute(XName.Get("ibExecID"))?.Value,
OpenCloseIndicator = e.Attribute(XName.Get("openCloseIndicator"))?.Value,
Isin = e.Attribute(XName.Get("isin"))?.Value,
Symbol = e.Attribute(XName.Get("symbol"))?.Value,
IsAPIOrder = e.Attribute(XName.Get("isAPIOrder"))?.Value,
Multiplier = ParseDecimal(e.Attribute(XName.Get("multiplier"))?.Value),
TransactionID =
long.TryParse(e.Attribute(XName.Get("transactionID"))?.Value, out var result)
? result
: 0,
IbOrderID = long.TryParse(e.Attribute(XName.Get("ibOrderID"))?.Value, out var resul)
? resul
: 0,
IbCommission = ParseDecimal(e.Attribute(XName.Get("ibCommission"))?.Value),
Conid = ParseDecimal(e.Attribute(XName.Get("conid"))?.Value),
Strike = ParseDecimal(e.Attribute(XName.Get("strike"))?.Value),
Taxes = ParseDecimal(e.Attribute(XName.Get("taxes"))?.Value),
FxRateToBase = ParseDecimal(e.Attribute(XName.Get("fxRateToBase"))?.Value),
Expiry = DateHelper.ParseDateTime(e.Attribute(XName.Get("expiry"))?.Value, "yyyyMMdd",
DateHelper.DefaultDateFormat),
OrderTime = DateHelper.ParseDateTime(e.Attribute(XName.Get("orderTime"))?.Value,
"yyyyMMdd;HHmmss", DateHelper.DefaultDateFormat),
SettleDateTarget = DateHelper.ParseDateTime(
e.Attribute(XName.Get("settleDateTarget"))?.Value, "yyyyMMdd",
DateHelper.DefaultDateFormat),
ReportDate = DateHelper.ParseDateTime(e.Attribute(XName.Get("reportDate"))?.Value,
"yyyyMMdd", DateHelper.DefaultDateFormat),
ListingExchange = e.Attribute(XName.Get("listingExchange"))?.Value,
UnderlyingSymbol = e.Attribute(XName.Get("underlyingSymbol"))?.Value,
ImportedFile = file
};
newTradeExeList.Add(newTradeExe);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while parsing TradeExe record. \nRow: {row}", ex, GetType().Name,
true);
}
}
}
try
{
_importJobRepository.AddRangeTradeExe(newTradeExeList);
_importJobRepository.SaveChanges();
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while adding TradeAs TradeExe to DB.", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
return(await Task.FromResult(FileStatus.Success));
}
private async Task <FileStatus> ProcessCashReport(ImportedFile file, Stream stream)
{
if (stream == null || stream.Length == 0)
{
return(await Task.FromResult(FileStatus.Failed));
}
var newTradeCashList = new List <TradeCash>();
using (var reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
var row = await reader.ReadLineAsync();
if (!row.Trim().StartsWith("<CashReportCurrency "))
{
continue;
}
try
{
var document = XDocument.Load(new MemoryStream(Encoding.UTF8.GetBytes(row)));
var el = document.Elements().First();
var reportDate = DateHelper.ParseDate(el.Attribute(XName.Get("fromDate"))?.Value);
if (reportDate == null)
{
throw new Exception(@"There is no fromDate");
}
await reader.ReadLineAsync();
if (!row.Trim().StartsWith("<CashReportCurrency "))
{
continue;
}
var doc = XDocument.Load(new MemoryStream(Encoding.UTF8.GetBytes(row)));
if (!doc.Elements().Any())
{
continue;
}
var e = doc.Elements().First();
if (!(e.Attribute(XName.Get("currency")).Value == "BASE_SUMMARY"))
{
continue;
}
var tradeAccId = GetOrCreateTradeAccountId(new TradeAccount
{
AccountName =
_fileNameMatcher.GetCorrectAccountId(e.Attribute(XName.Get("accountId"))?.Value),
AccountAlias = e.Attribute(XName.Get("acctAlias"))?.Value,
MasterAccountId = file.MasterAccountId,
ImportedFile = file
});
var newTradeCash = new TradeCash
{
TradeAccountId = tradeAccId,
ReportDate = reportDate.Value,
Deposits = ParseDecimal(e.Attribute(XName.Get("deposits"))?.Value),
Withdrawals = ParseDecimal(e.Attribute(XName.Get("withdrawals"))?.Value),
ImportedFile = file
};
newTradeCashList.Add(newTradeCash);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while parsing TradeCash record. \nRow: {row}", ex, GetType().Name,
true);
}
}
}
try
{
_importJobRepository.AddRangeTradeCash(newTradeCashList);
_importJobRepository.SaveChanges();
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while adding TradeCash records to DB.", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
return(await Task.FromResult(FileStatus.Success));
}
protected async override Task RunInternal()
{
SyncerInfoService.Log(GetType().Name, JobStatus.Started.ToString());
using (_copyJobRepository.BeginOperation())
{
try
{
var registeredFiles = _copyJobRepository
.RegisteredFilesQuery()
// order is important for file processing
.OrderByDescending(f => f.OriginalFileName.Contains(_fileNameMatcher.AcctStatusReportNonEcaSqlLike))
.ThenByDescending(f => f.OriginalFileName.Contains(_fileNameMatcher.AcctStatusReportSqlLike))
.ThenByDescending(f => f.OriginalFileName.Contains(_fileNameMatcher.SytossClientInfoSqlLike))
.ThenByDescending(f => f.FileCreateDate)
.ToList();
if (!registeredFiles.Any())
{
GlobalLogger.LogInfo("Job has not found new files for downloading. Waiting...", GetType().Name, true);
}
else
{
GlobalLogger.LogInfo($"Job has found new files: [{registeredFiles.Count()}] for downloading.", GetType().Name, true);
var totalCount = 0;
foreach (var file in registeredFiles)
{
totalCount++;
await TryLoadFileAsync(file);
GlobalLogger.LogInfo($"Amount copied: [{totalCount}] of [{registeredFiles.Count()}].", GetType().Name, true);
}
}
var loadedFailedFiles = _copyJobRepository.LoadedFailedFileList();
if (!loadedFailedFiles.Any())
{
GlobalLogger.LogInfo("Job has not found LOAD FAILED FILES. Waiting...", GetType().Name, true);
}
else
{
GlobalLogger.LogInfo($"Job has found LOAD FAILED FILES: [{loadedFailedFiles.Count()}].", GetType().Name, true);
var failedCount = 0;
foreach (var file in loadedFailedFiles)
{
failedCount++;
await TryLoadFileAsync(file);
GlobalLogger.LogInfo($"Amount of recopied (failed files): [{failedCount}] of [{loadedFailedFiles.Count()}].", GetType().Name, true);
}
}
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while copying files from ftp.", ex, GetType().Name, true);
}
}
SyncerInfoService.Log(GetType().Name, JobStatus.Stopped.ToString());
}
private async Task <FileStatus> ProcessNavFileReport(ImportedFile file, Stream stream)
{
if (stream == null || stream.Length == 0)
{
return(await Task.FromResult(FileStatus.Failed));
}
var newTradeNavList = new List <TradeNav>();
using (var reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
var row = await reader.ReadLineAsync();
if (!row.Trim().StartsWith("<EquitySummaryByReportDateInBase "))
{
continue;
}
try
{
var doc = XDocument.Load(new MemoryStream(Encoding.UTF8.GetBytes(row)));
if (!doc.Elements().Any())
{
continue;
}
var e = doc.Elements().First();
var reportDate = DateHelper.ParseDate(e.Attribute(XName.Get("reportDate"))?.Value);
if (reportDate == null)
{
continue;
}
var tradeAccId = GetOrCreateTradeAccountId(new TradeAccount
{
AccountName =
_fileNameMatcher.GetCorrectAccountId(e.Attribute(XName.Get("accountId"))?.Value),
AccountAlias = e.Attribute(XName.Get("acctAlias"))?.Value,
MasterAccountId = file.MasterAccountId,
ImportedFile = file
});
var newTradeNav = new TradeNav
{
TradeAccountId = tradeAccId,
ReportDate = reportDate.Value,
Total = ParseDecimal(e.Attribute(XName.Get("total"))?.Value),
Cash = ParseDecimal(e.Attribute(XName.Get("cash"))?.Value),
Stock = ParseDecimal(e.Attribute(XName.Get("stock"))?.Value),
Options = ParseDecimal(e.Attribute(XName.Get("options"))?.Value),
Commodities = ParseDecimal(e.Attribute(XName.Get("commodities"))?.Value),
InterestAccruals = ParseDecimal(e.Attribute(XName.Get("interestAccruals"))?.Value),
TotalLong = ParseDecimal(e.Attribute(XName.Get("totalLong"))?.Value),
CashLong = ParseDecimal(e.Attribute(XName.Get("cashLong"))?.Value),
StockLong = ParseDecimal(e.Attribute(XName.Get("stockLong"))?.Value),
OptionsLong = ParseDecimal(e.Attribute(XName.Get("optionsLong"))?.Value),
CommoditiesLong = ParseDecimal(e.Attribute(XName.Get("commoditiesLong"))?.Value),
InterestAccrualsLong = ParseDecimal(e.Attribute(XName.Get("interestAccrualsLong"))?.Value),
TotalShort = ParseDecimal(e.Attribute(XName.Get("totalShort"))?.Value),
CashShort = ParseDecimal(e.Attribute(XName.Get("cashShort"))?.Value),
StockShort = ParseDecimal(e.Attribute(XName.Get("stockShort"))?.Value),
OptionsShort = ParseDecimal(e.Attribute(XName.Get("optionsShort"))?.Value),
CommoditiesShort = ParseDecimal(e.Attribute(XName.Get("commoditiesShort"))?.Value),
InterestAccrualsShort = ParseDecimal(e.Attribute(XName.Get("interestAccrualsShort"))?.Value),
ImportedFile = file
};
newTradeNavList.Add(newTradeNav);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while parsing TradeNav record. \nRow: {row}", ex, GetType().Name,
true);
}
}
}
try
{
_importJobRepository.AddRangeTradeNav(newTradeNavList);
_importJobRepository.SaveChanges();
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while adding TradeNav records to DB.", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
return(await Task.FromResult(FileStatus.Success));
}
private async Task <FileStatus> ProcessSytossOpenPositions(ImportedFile file, Stream stream)
{
if (stream == null || stream.Length == 0)
{
return(await Task.FromResult(FileStatus.Failed));
}
var tradeAccSytossOpenPositionsRecords = new List <TradeSytossOpenPosition>();
using (var reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
var row = await reader.ReadLineAsync();
if (!row.Trim().StartsWith("<OpenPosition "))
{
continue;
}
try
{
var doc = XDocument.Load(new MemoryStream(Encoding.UTF8.GetBytes(row)));
if (!doc.Elements().Any())
{
continue;
}
var e = doc.Elements().First();
var tradeAccId = GetOrCreateTradeAccountId(new TradeAccount
{
AccountName =
_fileNameMatcher.GetCorrectAccountId(e.Attribute(XName.Get("accountId"))?.Value),
AccountAlias = e.Attribute(XName.Get("acctAlias"))?.Value,
MasterAccountId = file.MasterAccountId,
ImportedFile = file
});
var record = new TradeSytossOpenPosition
{
TradeAccountId = tradeAccId,
ReportDate = file.FileCreateDate,
Symbol = e.Attribute(XName.Get("symbol"))?.Value,
Position = ParseDecimal(e.Attribute(XName.Get("position"))?.Value),
MarkPrice = ParseDecimal(e.Attribute(XName.Get("markPrice"))?.Value),
CostBasisPrice = ParseDecimal(e.Attribute(XName.Get("costBasisPrice"))?.Value),
CostBasisMoney = ParseDecimal(e.Attribute(XName.Get("costBasisMoney"))?.Value),
PercentOfNav = ParseDecimal(e.Attribute(XName.Get("percentOfNAV"))?.Value),
FifoPnlUnrealized = ParseDecimal(e.Attribute(XName.Get("fifoPnlUnrealized"))?.Value),
Description = e.Attribute(XName.Get("description"))?.Value,
AssetCategory = e.Attribute(XName.Get("assetCategory"))?.Value,
UnderlyingListingExchange = e.Attribute(XName.Get("underlyingListingExchange"))?.Value,
Currency = e.Attribute(XName.Get("currency"))?.Value,
Isin = e.Attribute(XName.Get("isin"))?.Value,
Conid = ParseDecimal(e.Attribute(XName.Get("conid"))?.Value),
FxRateToBase = ParseDecimal(e.Attribute(XName.Get("fxRateToBase"))?.Value),
UnderlyingSymbol = e.Attribute(XName.Get("underlyingSymbol"))?.Value,
PutCall = e.Attribute(XName.Get("putCall"))?.Value,
Multiplier = ParseDecimal(e.Attribute(XName.Get("multiplier"))?.Value),
Strike = ParseDecimal(e.Attribute(XName.Get("strike"))?.Value),
Expiry = DateHelper.ParseDate(e.Attribute(XName.Get("expiry"))?.Value),
ImportedFile = file
};
tradeAccSytossOpenPositionsRecords.Add(record);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while parsing SytossOpenPositions record. \nRow: {row}", ex,
GetType().Name, true);
}
}
}
try
{
_importJobRepository.AddRangeOpenPositions(tradeAccSytossOpenPositionsRecords);
_importJobRepository.SaveChanges();
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while adding SytossOpenPosition.", ex, GetType().Name, true);
return(await Task.FromResult(FileStatus.Failed));
}
RefillTradeAccounts();
return(await Task.FromResult(FileStatus.Success));
}
protected override async Task RunInternal()
{
SyncerInfoService.Log(GetType().Name, JobStatus.Started.ToString());
using (_deliveryJobRepo.BeginOperation())
{
try
{
var filesToSend = _deliveryJobRepo.GetFilesToSendList();
if (!filesToSend.Any())
{
GlobalLogger.LogInfo("Job has not found new files for uploading. Waiting...", GetType().Name, true);
}
else
{
var totalCount = 0;
foreach (var fileUpload in filesToSend)
{
totalCount++;
var originalFileName = _deliveryJobRepo.GetOriginalFileName(fileUpload.Id);
var fileStream = _extractFileService.ExtractFile(originalFileName);
try
{
var response = await _restApiService.GetResponseOfUploadAsync(originalFileName, fileStream);
GlobalLogger.LogInfo($@"Response upload file: {originalFileName} to ZOHO api. | Status: {response.StatusCode} | ReasonPhrase: {response.ReasonPhrase} | RequestUri: {response.RequestMessage.RequestUri}", GetType().Name, true);
fileUpload.IsSent = response.IsSuccessStatusCode;
if (!response.IsSuccessStatusCode)
{
GlobalLogger.LogInfo($"File: {originalFileName} unsuccessfully uploaded | Status Code from ZOHO api: {response.StatusCode}", GetType().Name, true);
continue;
}
fileUpload.SentDate = DateTime.UtcNow;
GlobalLogger.LogInfo($"File: {originalFileName} successfully uploaded to ZOHO api.", GetType().Name, true);
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while upload file: {originalFileName} to ZOHO api.", ex, GetType().Name, true);
}
finally
{
_deliveryJobRepo.UpdateFileUpload(fileUpload);
await _deliveryJobRepo.SaveChangesAsync();
}
GlobalLogger.LogInfo($"Amount uploaded: [{totalCount}] of [{filesToSend.Count()}].", GetType().Name, true);
}
}
}
catch (Exception ex)
{
GlobalLogger.LogError($"Error while upload files to ZOHO api.", ex, this.GetType().Name, true);
}
}
SyncerInfoService.Log(GetType().Name, JobStatus.Stopped.ToString());
}
private async Task TryImportFile(ImportedFile currentFile)
{
var newFileStatus = FileStatus.Success;
try
{
if (currentFile != null)
{
GlobalLogger.LogInfo($"Start importing file: {currentFile.OriginalFileName} | Status: {currentFile.FileStatus}", GetType().Name, true);
var streamFile = _extractFileService.ExtractFile(currentFile.OriginalFileName);
if (_fileNameMatcher.IsFeesDataFile(currentFile.OriginalFileName))
{
newFileStatus = await ProcessFeesFileReport(currentFile, streamFile);
}
else if (_fileNameMatcher.IsAcctStatusReportNonEca(currentFile.OriginalFileName))
{
newFileStatus = await ProcessAcctStatusReportFileReport(
currentFile,
"AccountStatusReport",
_fileNameMatcher.AcctStatusReportNonEcaSqlLike,
streamFile);
}
else if (_fileNameMatcher.IsNavRegex(currentFile.OriginalFileName))
{
newFileStatus = await ProcessNavFileReport(currentFile, streamFile);
}
else if (_fileNameMatcher.IsAcctStatusReport(currentFile.OriginalFileName))
{
newFileStatus = await ProcessAcctStatusReportFileReport(
currentFile,
"ECAAccountStatusReport",
_fileNameMatcher.AcctStatusReportSqlLike,
streamFile);
}
else if (_fileNameMatcher.IsCashReport(currentFile.OriginalFileName))
{
newFileStatus = await ProcessCashReport(currentFile, streamFile);
}
else if (_fileNameMatcher.IsTradeAsReport(currentFile.OriginalFileName))
{
newFileStatus = await ProcessTradesAsReport(currentFile, streamFile);
}
else if (_fileNameMatcher.IsSytossClientInfo(currentFile.OriginalFileName))
{
newFileStatus = await ProcessSytossClientInfoReport(currentFile, streamFile);
}
else if (_fileNameMatcher.IsSytossOpenPositions(currentFile.OriginalFileName))
{
newFileStatus = await ProcessSytossOpenPositions(currentFile, streamFile);
}
else if (_fileNameMatcher.IsTradeExeReport(currentFile.OriginalFileName))
{
newFileStatus = await ProcessTradesExeReport(currentFile, streamFile);
}
else if (_fileNameMatcher.IsInterestAccruaReport(currentFile.OriginalFileName))
{
newFileStatus = await ProcessTradeInterestAccruaReport(currentFile, streamFile);
}
else if (_fileNameMatcher.IsTradeCommissionsDetReport(currentFile.OriginalFileName))
{
newFileStatus = await ProcessTradeCommissionsReport(currentFile, streamFile);
}
}
GlobalLogger.LogInfo($"File: {currentFile.OriginalFileName} has been imported.", GetType().Name, true);
}
catch (Exception ex)
{
newFileStatus = FileStatus.Failed;
GlobalLogger.LogError($"Error while importing file {currentFile?.OriginalFileName}.", ex, GetType().Name, true);
}
finally
{
if (currentFile != null)
{
currentFile.FileState = FileState.Imported;
currentFile.FileStatus = newFileStatus;
if (newFileStatus == FileStatus.Success)
{
currentFile.ImportedDate = DateTime.UtcNow;
}
_importJobRepository.UpdateImportedFile(currentFile);
_importJobRepository.SaveChanges();
}
else
{
GlobalLogger.LogInfo("Job has not found new files for importing. Waiting...", GetType().Name, true);
}
}
}