public void Configuration(IAppBuilder app)
{
// Check to see if the default database exists, if not, create it
using (DataFlowDbContext ctx = new DataFlowDbContext())
{
bool exists = ctx.Database
.SqlQuery <int?>(@"
SELECT 1 FROM sys.tables AS T
INNER JOIN sys.schemas AS S ON T.schema_id = S.schema_id
WHERE S.Name = 'dbo' AND T.Name = 'Agents'")
.SingleOrDefault() != null;
Configuration config = new Configuration();
if (!exists)
{
// Auto-migrate the DataFlow.Common entity model to the database
DbMigrator migrator = new DbMigrator(config);
migrator.Update();
}
if (ctx.Entities.Count() == 0 || ctx.EdfiDictionary.Count() == 0 || ctx.FileStatuses.Count() == 0)
{
config.ForceSeed(ctx);
}
}
ConfigureAuth(app);
}
private static bool DoesFileExistInLog(int agentId, string file)
{
bool fileFound = false;
using (DataFlowDbContext ctx = new DataFlowDbContext())
{
int fileCount = ctx.Files
.Where(f => f.AgentId == agentId)
.Where(f => f.FileName == file)
.Count();
if (fileCount == 0)
{
fileFound = false;
}
else
{
fileFound = true;
}
}
_log.Info("DoesFileExistInLog: {0} is: {1}", file, fileFound.ToString());
return(fileFound);
}
public async Task InsertBootstrapData()
{
Assert.Inconclusive();
await Task.Yield();
using (DataFlowDbContext ctx = new DataFlowDbContext())
{
await DataFlow.Server.TransformLoad.Program.InsertBootrapData(ctx);
}
}
public AgentService(DataFlowDbContext dataFlowDbContext, ILogService logService)
{
this.dataFlowDbContext = dataFlowDbContext;
this.LogService = logService;
if (LogService != null)
{
LogService.Name = GetType().FullName;
}
}
public ActionResult Retry(int id)
{
using (var ctx = new DataFlowDbContext())
{
var file = ctx.Files.Find(id);
file.Status = FileStatusEnum.RETRY;
file.UpdateDate = DateTime.Now;
ctx.SaveChanges();
}
return(RedirectToAction("Index"));
}
private static void LogFile(int agentId, string fileName, string URL, string status, int rows)
{
using (DataFlowDbContext ctx = new DataFlowDbContext())
{
Models.File fileLog = new Models.File();
fileLog.AgentId = agentId;
fileLog.FileName = fileName;
fileLog.Url = URL;
fileLog.Rows = rows;
fileLog.Status = status;
fileLog.CreateDate = DateTime.Now;
ctx.Files.Add(fileLog);
ctx.SaveChanges();
}
}
public static void Main(string[] args)
{
// Force TLS 1.2 as per Ed-Fi ODS-2403 -- https://tracker.ed-fi.org/browse/ODS-2403
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
_log.Info("File Transport Janitor starting");
if (HaveConfigurationValues())
{
using (DataFlowDbContext ctx = new DataFlowDbContext())
{
var agents = ctx.Agents.Where(agent => agent.Enabled == true && (agent.AgentTypeCode == AgentTypeCodeEnum.SFTP || agent.AgentTypeCode == AgentTypeCodeEnum.FTPS));
foreach (var agent in agents)
{
_log.Info("Processing agent name: {0}", agent.Name);
List <string> fileList = GetFileList(agent);
_log.Info("Items to process: {0}", fileList.Count);
foreach (string file in fileList)
{
_log.Info("Processing file: {0}", file);
// Check the file log to see if the file already exists, if not, upload to file storage
if (!DoesFileExistInLog(agent.Id, file))
{
TransferFileToStorage(agent, file);
}
}
agent.LastExecuted = DateTime.Now;
}
ctx.SaveChanges();
}
}
_log.Info("File Transport Janitor exiting");
if (System.Diagnostics.Debugger.IsAttached)
{
Console.WriteLine("Press any key to exit...");
Console.ReadLine();
}
}
public static void ClassCleanup()
{
using (var ctx = new DataFlowDbContext())
{
AgentAgentChrome aac = ctx.AgentAgentChromes.Find(_agentAgentChromeId);
ctx.AgentAgentChromes.Remove(aac);
AgentChrome chrome = ctx.AgentChromes.Find(_agentChromeId);
ctx.AgentChromes.Remove(chrome);
Agent agent = ctx.Agents.Find(_agentId);
// Remove the directory with the agent temp file sample.csv
System.IO.Directory.Delete(PathUtility.EnsureTrailingSlash(ConfigurationManager.AppSettings["ShareName"]) + agent.Queue.ToString(), true);
ctx.Agents.Remove(agent);
ctx.SaveChanges();
}
}
public void FillSwaggerMetadata(string apiServerUrl)
{
System.Net.ServicePointManager.ServerCertificateValidationCallback = (senderLocal, certificate, chain, sslPolicyErrors) => true;
using (DataFlowDbContext ctx = new DataFlowDbContext())
{
WebClient client = new WebClient();
string baseUrl = Common.Helpers.UrlUtility.GetUntilOrEmpty(apiServerUrl.Trim(), "/api/");
foreach (Entity entity in ctx.Entities)
{
string url = baseUrl + entity.Url;
string metadata = client.DownloadString(url);
entity.Metadata = metadata;
}
ctx.SaveChanges();
}
}
// This method validates the uuid and token is related to the agent, if so, return the agent object
public Agent GetAgent(Guid uuid, Guid token, int agent_id)
{
Agent returnAgent = null;
using (var ctx = new DataFlowDbContext())
{
AgentChrome chrome = ctx.AgentChromes.Where(ac => ac.AgentUuid == uuid && ac.AccessToken == token).FirstOrDefault();
if (chrome != null)
{
var aac = ctx.AgentAgentChromes.Where(aacc => aacc.AgentChromeId == chrome.Id && aacc.AgentId == agent_id).Include(aacc => aacc.Agent.AgentSchedules).FirstOrDefault();
if (aac != null && aac.Agent != null)
{
returnAgent = aac.Agent;
}
}
}
return(returnAgent);
}
public List <AgentResponse> Agents([FromBody] AgentMessage message)
{
List <AgentResponse> response = new List <AgentResponse>();
try
{
using (var ctx = new DataFlowDbContext())
{
AgentChrome agentChrome = ctx.AgentChromes.Where(ac => ac.AgentUuid == message.uuid && ac.AccessToken == message.token).FirstOrDefault();
if (agentChrome != null)
{
List <AgentAgentChrome> agentAgentChromes = ctx.AgentAgentChromes.Where(aacc => aacc.AgentChromeId == agentChrome.Id && aacc.Agent.Enabled == true).Include(aacc => aacc.Agent.AgentSchedules).ToList();
foreach (var agentAgentChrome in agentAgentChromes)
{
AgentResponse responseAgent = new AgentResponse();
responseAgent.agent_id = agentAgentChrome.Agent.Id;
responseAgent.name = agentAgentChrome.Agent.Name;
responseAgent.action = agentAgentChrome.Agent.AgentAction;
responseAgent.parameters = agentAgentChrome.Agent.Custom;
responseAgent.url = agentAgentChrome.Agent.Url;
responseAgent.loginUrl = agentAgentChrome.Agent.LoginUrl;
responseAgent.schedule = new List <AgentScheduleResponse>();
foreach (var sch in agentAgentChrome.Agent.AgentSchedules)
{
AgentScheduleResponse schedule = new AgentScheduleResponse(sch.Day, sch.Hour, sch.Minute);
responseAgent.schedule.Add(schedule);
}
response.Add(responseAgent);
}
}
}
}
catch (System.Exception ex)
{
_logger.Error(ex, "Unexpected error in Agents");
throw new HttpResponseException(HttpStatusCode.InternalServerError);
}
return(response);
}
public static void ClassInit(TestContext context)
{
using (var ctx = new DataFlowDbContext())
{
Agent agent = new Agent();
agent.AgentTypeCode = Common.Enums.AgentTypeCodeEnum.Chrome;
agent.Name = "Unit Test Chrome Agent";
agent.Url = "http://localhost";
agent.LoginUrl = "http://localhost/login.php";
agent.Enabled = true;
agent.Queue = Guid.NewGuid();
agent.Created = DateTime.Now;
ctx.Agents.Add(agent);
ctx.SaveChanges();
_agentId = agent.Id;
}
}
public async Task EdFiApiAuthentication()
{
using (DataFlowDbContext ctx = new DataFlowDbContext())
{
string authorizeUrl = DataFlow.Server.TransformLoad.Program.GetAuthorizeUrl(ctx);
Assert.IsFalse(string.IsNullOrWhiteSpace(authorizeUrl), string.Format("{0} is Empty", nameof(authorizeUrl)));
string accessTokenUrl = DataFlow.Server.TransformLoad.Program.GetAccessTokenUrl(ctx);
Assert.IsFalse(string.IsNullOrWhiteSpace(accessTokenUrl), string.Format("{0} is Empty", nameof(accessTokenUrl)));
string clientId = DataFlow.Server.TransformLoad.Program.GetApiClientId(ctx);
Assert.IsFalse(string.IsNullOrWhiteSpace(clientId), string.Format("{0} is Empty", nameof(clientId)));
string clientSecret = DataFlow.Server.TransformLoad.Program.GetApiClientSecret(ctx);
Assert.IsFalse(string.IsNullOrWhiteSpace(clientSecret), string.Format("{0} is Empty", nameof(clientSecret)));
string authCode = await DataFlow.Server.TransformLoad.Program.RetrieveAuthorizationCode(authorizeUrl, clientId : clientId);
Assert.IsFalse(string.IsNullOrWhiteSpace(authCode), string.Format("{0} is Empty", nameof(authCode)));
string accessToken = await DataFlow.Server.TransformLoad.Program.RetrieveAccessToken(accessTokenUrl, clientId, clientSecret, authCode);
Assert.IsFalse(string.IsNullOrWhiteSpace(accessToken), string.Format("{0} is Empty", nameof(accessToken)));
}
}
public HttpResponseMessage Register([FromBody] AgentMessage message)
{
try
{
using (var ctx = new DataFlowDbContext())
{
AgentChrome chrome = ctx.AgentChromes.FirstOrDefault(ac => ac.AgentUuid == message.uuid);
if (chrome != null)
{
_logger.Error(String.Format("Duplicate Chrome Agent registration requested for uuid = {0}", message.uuid));
return(new HttpResponseMessage()
{
StatusCode = HttpStatusCode.BadRequest
});
}
else
{
chrome = new AgentChrome();
chrome.AgentUuid = message.uuid;
chrome.AccessToken = Guid.NewGuid();
chrome.Created = DateTime.Now;
ctx.AgentChromes.Add(chrome);
ctx.SaveChanges();
message.token = chrome.AccessToken;
_logger.Info(String.Format("Registered new Chrome Agent with uuid = {0}", message.uuid));
}
}
}
catch (Exception ex)
{
_logger.Error(ex, "Unexpected error in Register");
return(new HttpResponseMessage()
{
StatusCode = HttpStatusCode.InternalServerError
});
}
return(Request.CreateResponse((HttpStatusCode)200, message));
}
public HttpResponseMessage Data([FromBody] AgentMessage message)
{
HttpStatusCode statusCode = HttpStatusCode.InternalServerError;
try
{
using (var ctx = new DataFlowDbContext())
{
Agent agent = GetAgent(message.uuid, message.token, message.agent_id);
if (agent != null && message.data != null && message.filename != null)
{
byte[] dataArray = Convert.FromBase64String(message.data);
if (dataArray != null)
{
System.IO.MemoryStream stream = new System.IO.MemoryStream(dataArray);
AgentService svc = new AgentService(ctx, _loggerService);
Tuple <bool, string> result = svc.UploadFile(message.filename, stream, agent);
if (result.Item1)
{
statusCode = HttpStatusCode.OK;
}
}
}
}
}
catch (Exception ex)
{
_logger.Error(ex, "Unexpected error in Data");
}
return(new HttpResponseMessage()
{
StatusCode = statusCode
});
}
public void RegisterAgent()
{
ChromeExtensionController cec = new ChromeExtensionController();
cec.Request = new HttpRequestMessage();
cec.Request.SetConfiguration(new HttpConfiguration());
AgentMessage message = new AgentMessage();
message.uuid = Guid.NewGuid();
HttpResponseMessage response = cec.Register(message);
var content = response.Content;
Assert.IsNotNull(content);
Assert.IsInstanceOfType(content, typeof(ObjectContent <AgentMessage>));
AgentMessage responseMessage = (AgentMessage)((ObjectContent <AgentMessage>)content).Value;
Assert.IsNotNull(responseMessage.token);
Assert.AreNotEqual(Guid.Parse("00000000-0000-0000-0000-000000000000"), responseMessage.token);
_agentGuid = responseMessage.uuid;
_agentToken = responseMessage.token;
// If we've gotten this far, we've registered a new AgentChrome, let's associate it with the agent
using (var ctx = new DataFlowDbContext())
{
AgentChrome ac = ctx.AgentChromes.Where(chr => chr.AgentUuid == _agentGuid && chr.AccessToken == _agentToken).FirstOrDefault();
AgentAgentChrome aac = new AgentAgentChrome();
aac.AgentChromeId = ac.Id;
aac.AgentId = _agentId;
ctx.AgentAgentChromes.Add(aac);
ctx.SaveChanges();
_agentChromeId = ac.Id;
_agentAgentChromeId = aac.Id;
}
}
public AccountController(DataFlowDbContext dataFlowDbContext, ApplicationUserManager userManager, ApplicationSignInManager signInManager, IBaseServices baseService) : base(baseService)
{
this.dataFlowDbContext = dataFlowDbContext;
UserManager = userManager;
SignInManager = signInManager;
}
public LogController(DataFlowDbContext dataFlowDbContext, IBaseServices baseService) : base(baseService)
{
this.dataFlowDbContext = dataFlowDbContext;
}
public SchoolController(DataFlowDbContext dataFlowDbContext, EdFiService edFiService, IBaseServices baseService) : base(baseService)
{
this.dataFlowDbContext = dataFlowDbContext;
this.edFiService = edFiService;
}
public ConfigurationService(DataFlowDbContext dataFlowDbContext, ICacheService cacheService)
{
this.dataFlowDbContext = dataFlowDbContext;
this.cacheService = cacheService;
}
public EdFiService(DataFlowDbContext dataFlowDbContext, IConfigurationService configurationService)
{
this.dataFlowDbContext = dataFlowDbContext;
this.configurationService = configurationService;
}
static void Main(string[] args)
{
_log.Info("Reporting and Cleanup Janitor starting");
EmailMessage emailMessage = new EmailMessage()
{
EmailDate = DateTime.Now.ToShortDateString()
};
var smtpSection = (SmtpSection)SysConfig.ConfigurationManager.GetSection("system.net/mailSettings/smtp");
string fromEmail = smtpSection.From;
using (var ctx = new DataFlowDbContext())
{
try
{
// Get processed deleted files
string daysToKeepFiles = SysConfig.ConfigurationManager.AppSettings["DaysToKeepFiles"];
int days;
if (daysToKeepFiles != null && daysToKeepFiles != "0" && int.TryParse(daysToKeepFiles, out days))
{
DateTime deleteFileDate = DateTime.Now.AddDays(-days);
List <File> deletedFiles = ctx.Files.Where(f => f.CreateDate <= deleteFileDate && f.Status != FileStatusEnum.DELETED).ToList();
emailMessage.DeleteDays = days;
emailMessage.DeletedFiles = deletedFiles;
foreach (var file in deletedFiles)
{
try
{
string fullFilePath = new System.Uri(file.Url).LocalPath;
System.IO.File.Delete(fullFilePath);
file.Status = FileStatusEnum.DELETED;
}
catch (Exception deleteEx)
{
_log.Error(deleteEx, "Error deleting file: {0}", file.FileName);
}
}
ctx.SaveChanges();
}
// Send email message
DateTime filterDate = DateTime.MinValue;
Configuration lastRan = ctx.Configurations.Where(c => c.Key == JANITOR_REPORT_LAST).FirstOrDefault();
if (lastRan == null)
{
lastRan = new Configuration()
{
Key = JANITOR_REPORT_LAST
};
ctx.Configurations.Add(lastRan);
}
else
{
filterDate = DateTime.Parse(lastRan.Value);
}
lastRan.Value = DateTime.Now.ToString();
List <File> files = ctx.Files.Where(f => (f.CreateDate >= filterDate || f.UpdateDate >= filterDate) && f.Status != FileStatusEnum.DELETED).Include(f => f.Agent).ToList();
List <AspNetUser> users = ctx.AspNetUsers.ToList();
MailAddressCollection emails = new MailAddressCollection();
foreach (AspNetUser user in users)
{
emails.Add(new MailAddress(user.Email));
}
emailMessage.Files = files;
if (emailMessage.Files != null || emailMessage.DeletedFiles != null)
{
var email = Email
.From(fromEmail)
.To(emails)
.Subject(String.Format("Data Flow Status Update for {0}", emailMessage.EmailDate))
.UsingTemplateFromFile("./template.cshtml", emailMessage);
_log.Info("Sending email report to {0} for {1} of files", emails.ToString(), files.Count);
email.Send();
}
else
{
_log.Info("No email report to send today (no processed or deleted files).");
}
ctx.SaveChanges();
}
catch (Exception ex)
{
_log.Error(ex, "Unexpected error in ReportingCleanup service");
}
}
_log.Info("Reporting and Cleanup Janitor ending");
if (System.Diagnostics.Debugger.IsAttached)
{
Console.WriteLine("Press any key to exit...");
Console.ReadLine();
}
}
public AgentController(DataFlowDbContext dataFlowDbContext, AgentService agentService, IBaseServices baseService) : base(baseService)
{
this.dataFlowDbContext = dataFlowDbContext;
this.agentService = agentService;
this.EncryptionKey = WebConfigAppSettingsService.GetSetting <string>(Constants.AppSettingEncryptionKey);
}
public BootstrapDataController(DataFlowDbContext dataFlowDbContext, IBaseServices baseServices) : base(baseServices)
{
this.dataFlowDbContext = dataFlowDbContext;
}
public DataMapperController(DataFlowDbContext dataFlowDbContext, EdFiMetadataProcessor edFiMetadataProcessor,
IBaseServices baseService) : base(baseService)
{
this.dataFlowDbContext = dataFlowDbContext;
this.edFiMetadataProcessor = edFiMetadataProcessor;
}