// list directory // etl each file separately // load file // match all four types & insert (if necessary); match in memory (performance) // match user & insert (if necessary); via sproc (Cache issue) // load session and values into database // delete file when done & no exceptions public static void ImportMessagesFromDirectory(string directory, bool KeepFile) { using (var context = CollectorRepository.CreateContext()) { CollectorRepository repo = new CollectorRepository(); repo.Context = context; foreach (string filename in Directory.EnumerateFiles(directory, "*.xml.gz", SearchOption.TopDirectoryOnly)) { UsageDataMessage message = null; try { message = FileImporter.ReadMessage(filename); } catch (System.Exception ex) { Console.WriteLine("Failed to read file {0}, exception {1}", filename, ex.Message); continue; } StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo); processor.ProcessMessage(); } } }
public static void ImportSingleMessage(string filename) { UsageDataMessage message = FileImporter.ReadMessage(filename); using (var context = CollectorRepository.CreateContext()) { CollectorRepository repo = new CollectorRepository(); repo.Context = context; StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo); processor.ProcessMessage(); } }
public override bool Execute() { bool result = true; if ((null == messagesToImport) || (0 == messagesToImport.Length)) { // messagesFailedToImport is already newed as an empty list return true; } if (null == connectionString) { Log.LogError("Connection String for ADO.NET EF cannot be empty"); return false; } // Import logic built for MSBuild Stopwatch watch = Stopwatch.StartNew(); using (var context = CollectorRepository.CreateContext(connectionString.ItemSpec)) { CollectorRepository repo = new CollectorRepository(); repo.Context = context; foreach (ITaskItem msgFilename in messagesToImport) { UsageDataMessage message = null; try { message = FileImporter.ReadMessage(msgFilename.ItemSpec); } catch (System.Exception ex) { Log.LogErrorFromException(ex); messagesFailedToImport.Add(new TaskItem(msgFilename.ItemSpec)); continue; } if (ignoreSessionsTooFarFromFileDate) { // Acceptable sessions are between 1.5 months old and 3 days into the future. // All other sessions indicate a horribly wrong system time on the user's machine and will be ignored. DateTime fileDate = System.IO.File.GetLastWriteTimeUtc(msgFilename.ItemSpec); message.Sessions.RemoveAll(s => s.StartTime < fileDate.AddDays(-45) || s.StartTime > fileDate.AddDays(3)); } try { StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo); processor.ProcessMessage(); } catch (System.Exception ex) { Log.LogErrorFromException(ex); messagesFailedToImport.Add(new TaskItem(msgFilename.ItemSpec)); } } } watch.Stop(); Log.LogMessage("Imported " + (messagesToImport.Length - messagesFailedToImport.Count) + " messages in " + watch.Elapsed); return result; }