public WebCrawler(ICrawlingStats crawlingStats, IResultWriter resultWriter, IClock clock)
{
_crawlingStats = crawlingStats;
_resultWriter = resultWriter;
_clock = clock;
_webCrawler = new PoliteWebCrawler();
_webCrawler.PageCrawlCompletedAsync += ProcessPageCrawlCompleted;
_startCrawlingTime = _clock.FormattedCurrentTime();
//_resultFilePath = System.Configuration.ConfigurationManager.AppSettings["ResultFileName"];
}
public MapRenderModule(IRenderQueue queue, IMapRenderer renderer, IJobPrioritizer prioritizer, IResultWriter writer, IResultReader reader, IConfigService config, IWorldFinder finder, IWorldArchivist archivist)
{
_queue = queue;
_renderer = renderer;
_prioritizer = prioritizer;
_writer = writer;
_reader = reader;
_config = config;
_finder = finder;
_archivist = archivist;
ensureOverviewerOutput();
}
public EventProcessingProjectionProcessingPhase(
CoreProjection coreProjection,
Guid projectionCorrelationId,
IPublisher publisher,
IPublisher inputQueue,
ProjectionConfig projectionConfig,
Action updateStatistics,
IProjectionStateHandler projectionStateHandler,
PartitionStateCache partitionStateCache,
bool definesStateTransform,
string projectionName,
ILogger logger,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager coreProjectionCheckpointManager,
StatePartitionSelector statePartitionSelector,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
bool useCheckpoints,
bool stopOnEof,
bool isBiState,
bool orderedPartitionProcessing,
IEmittedStreamsTracker emittedStreamsTracker)
: base(
publisher,
inputQueue,
coreProjection,
projectionCorrelationId,
coreProjectionCheckpointManager,
projectionConfig,
projectionName,
logger,
zeroCheckpointTag,
partitionStateCache,
resultWriter,
updateStatistics,
subscriptionDispatcher,
readerStrategy,
useCheckpoints,
stopOnEof,
orderedPartitionProcessing,
isBiState,
emittedStreamsTracker)
{
_projectionStateHandler = projectionStateHandler;
_definesStateTransform = definesStateTransform;
_statePartitionSelector = statePartitionSelector;
_isBiState = isBiState;
_stopwatch = new Stopwatch();
}
public ParallelQueryMasterProjectionProcessingPhase(
CoreProjection coreProjection, Guid projectionCorrelationId, IPublisher publisher,
ProjectionConfig projectionConfig, Action updateStatistics, PartitionStateCache partitionStateCache,
string name, ILogger logger, CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager, ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy, IResultWriter resultWriter, bool checkpointsEnabled, bool stopOnEof,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher)
: base(
publisher, coreProjection, projectionCorrelationId, checkpointManager, projectionConfig, name, logger,
zeroCheckpointTag, partitionStateCache, resultWriter, updateStatistics, subscriptionDispatcher,
readerStrategy, checkpointsEnabled, stopOnEof)
{
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
}
protected EventSubscriptionBasedProjectionProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
ICoreProjectionForProcessingPhase coreProjection,
Guid projectionCorrelationId,
ICoreProjectionCheckpointManager checkpointManager,
ProjectionConfig projectionConfig,
string projectionName,
ILogger logger,
CheckpointTag zeroCheckpointTag,
PartitionStateCache partitionStateCache,
IResultWriter resultWriter,
Action updateStatistics,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
bool useCheckpoints,
bool stopOnEof,
bool orderedPartitionProcessing,
bool isBiState,
IEmittedStreamsTracker emittedStreamsTracker,
bool enableContentTypeValidation)
{
_publisher = publisher;
_inputQueue = inputQueue;
_coreProjection = coreProjection;
_projectionCorrelationId = projectionCorrelationId;
_checkpointManager = checkpointManager;
_projectionConfig = projectionConfig;
_projectionName = projectionName;
_logger = logger;
_zeroCheckpointTag = zeroCheckpointTag;
_partitionStateCache = partitionStateCache;
_resultWriter = resultWriter;
_updateStatistics = updateStatistics;
_processingQueue = new CoreProjectionQueue(publisher,
projectionConfig.PendingEventsThreshold,
orderedPartitionProcessing);
_processingQueue.EnsureTickPending += EnsureTickPending;
_subscriptionDispatcher = subscriptionDispatcher;
_readerStrategy = readerStrategy;
_useCheckpoints = useCheckpoints;
_stopOnEof = stopOnEof;
_isBiState = isBiState;
_progressResultWriter = new ProgressResultWriter(this, _resultWriter);
_inutQueueEnvelope = new PublishEnvelope(_inputQueue);
_emittedStreamsTracker = emittedStreamsTracker;
_enableContentTypeValidation = enableContentTypeValidation;
}
public DirectoryHandler(string path,
IFileHandler handler,
IResultWriter writer,
IPathCalculator calculator,
IPrinter printer)
{
if (string.IsNullOrWhiteSpace(path) || !Directory.Exists(path))
{
throw new ArgumentException(nameof(path));
}
_fileHandler = handler ?? throw new ArgumentNullException(nameof(handler));
_resultWriter = writer ?? throw new ArgumentNullException(nameof(writer));
_pathCalculator = calculator ?? throw new ArgumentNullException(nameof(calculator));
_startPath = Path.GetFullPath(path);
_printer = printer;
}
/// <summary>
/// Конструктор класса.
/// </summary>
public MainForm(IResultWriter resultWriter)
{
InitializeComponent();
tlTip.SetToolTip(btnExportToExcel, "Экспортирует таблицы с результатами в MS Excel.");
tlTip.SetToolTip(btnExportToWord, "Экспортирует текстовые результаты (тройные и четверные встречаемости) в MS Word.");
Application.Idle += Idle;
_resultWriter = resultWriter;
AllWords = new List <string>();
Markems = new List <string>();
Definitions = new List <string>();
FreeAssociations = new List <string>();
DirectAssociations = new List <string>();
Similarities = new List <string>();
Opposities = new List <string>();
}
public TestSessionRunner(Config config, IHttpClient httpClient, IResultWriter resultWriter)
{
if (config == null)
throw new ArgumentNullException("config");
if (httpClient == null)
throw new ArgumentNullException("httpClient");
if (resultWriter == null)
throw new ArgumentNullException("resultWriter");
_config = config;
_httpClient = httpClient;
_resultWriter = resultWriter;
_currentResults = new List<TestCaseResult>();
}
public CopyResultWriter(
Action <InfoMessage> addInfoMessage,
IProvider destinationProvider,
ConnectionBase destinationConnection,
string tableName,
Action <IDbTransaction> setTransaction,
CancellationToken cancellationToken)
{
_logResultWriter = new LogResultWriter(addInfoMessage);
_addInfoMessage = addInfoMessage;
_destinationProvider = destinationProvider;
_canConvertCommandToString = destinationProvider.CanConvertCommandToString;
_destinationConnection = destinationConnection;
_tableName = tableName;
_setTransaction = setTransaction;
_cancellationToken = cancellationToken;
}
public ParallelQueryMasterProjectionProcessingPhase(
CoreProjection coreProjection,
Guid projectionCorrelationId,
IPublisher publisher,
IPublisher inputQueue,
ProjectionConfig projectionConfig,
Action updateStatistics,
IProjectionStateHandler stateHandler,
PartitionStateCache partitionStateCache,
string name,
ILogger logger,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
bool checkpointsEnabled,
bool stopOnEof,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
IEmittedStreamsTracker emittedStreamsTracker)
: base(
publisher,
inputQueue,
coreProjection,
projectionCorrelationId,
checkpointManager,
projectionConfig,
name,
logger,
zeroCheckpointTag,
partitionStateCache,
resultWriter,
updateStatistics,
subscriptionDispatcher,
readerStrategy,
checkpointsEnabled,
stopOnEof,
orderedPartitionProcessing: true,
isBiState: false,
emittedStreamsTracker: emittedStreamsTracker)
{
_stateHandler = stateHandler;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
_spoolProcessingWorkItems = new Dictionary <Guid, SpoolStreamProcessingWorkItem>();
}
static List <string> GetFullFilePath(DirectoryInfo dir, string filename, IResultWriter writer)
{
Regex reg = new Regex(filename);
List <string> fileinfo = new List <string>();
//string file = String.Empty;
try
{
fileinfo = Directory.GetFiles(dir.FullName, @"*.*")
.Where(path => reg.IsMatch(path))
.ToList();
}
catch (Exception e)
{
writer.WriteVerbose("Couldn't locate file " + filename);
}
return(fileinfo);
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
IEmittedStreamsTracker emittedStreamsTracker)
{
var statePartitionSelector = CreateStatePartitionSelector();
var orderedPartitionProcessing = _sourceDefinition.ByStreams && _sourceDefinition.IsBiState;
return(new EventProcessingProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_sourceDefinition.DefinesStateTransform,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
statePartitionSelector,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_sourceDefinition.IsBiState,
orderedPartitionProcessing: orderedPartitionProcessing,
emittedStreamsTracker: emittedStreamsTracker,
enableContentTypeValidation: _enableContentTypeValidation));
}
public SqlBulkCopyResultWriter(
Action <InfoMessage> addInfoMessage,
IProvider destinationProvider,
ConnectionBase destinationConnection,
string tableName,
Action <IDbTransaction> setTransaction,
CancellationToken cancellationToken)
{
Assert.IsTrue(destinationProvider.DbProviderFactory == SqlClientFactory.Instance);
_destinationSqlConnection = (SqlConnection)destinationConnection.Connection;
_logResultWriter = new LogResultWriter(addInfoMessage);
_addInfoMessage = addInfoMessage;
_destinationProvider = destinationProvider;
_canConvertCommandToString = destinationProvider.CanConvertCommandToString;
_destinationConnection = destinationConnection;
_tableName = tableName;
_setTransaction = setTransaction;
_cancellationToken = cancellationToken;
}
public SpoolStreamProcessingWorkItem(
ISpoolStreamWorkItemContainer container,
IPublisher publisher,
IResultWriter resultWriter,
ParallelProcessingLoadBalancer loadBalancer,
EventReaderSubscriptionMessage.CommittedEventReceived message,
SlaveProjectionCommunicationChannels slaves,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
long limitingCommitPosition,
Guid subscriptionId,
bool definesCatalogTransform)
: base(Guid.NewGuid())
{
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (resultWriter == null)
{
throw new ArgumentNullException("resultWriter");
}
if (slaves == null)
{
throw new ArgumentNullException("slaves");
}
if (spoolProcessingResponseDispatcher == null)
{
throw new ArgumentNullException("spoolProcessingResponseDispatcher");
}
_container = container;
_publisher = publisher;
_resultWriter = resultWriter;
_loadBalancer = loadBalancer;
_message = message;
_slaves = slaves;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
_limitingCommitPosition = limitingCommitPosition;
_subscriptionId = subscriptionId;
_definesCatalogTransform = definesCatalogTransform;
}
private static void HandleCorrelatedEvents(IResultWriter writer, string eventInstanceId, QueryParametersWrapper queryParameters)
{
if (string.IsNullOrWhiteSpace(eventInstanceId))
{
EventStoreLogger.Logger.LogWarning("HandleCorrelatedEvents:: Event Instance id in param is empty/null");
OnUnsupportedUri(writer);
return;
}
var eventInstanceGuid = Guid.Empty;
if (!Guid.TryParse(eventInstanceId, out eventInstanceGuid))
{
EventStoreLogger.Logger.LogWarning("HandleCorrelatedEvents:: EventInstance ID : {0} Is not a valid Guid", eventInstanceId);
OnUnsupportedUri(writer);
return;
}
if (eventInstanceGuid == Guid.Empty)
{
EventStoreLogger.Logger.LogWarning("HandleCorrelatedEvents:: Empty Guid for Event Instance Id");
OnUnsupportedUri(writer);
return;
}
var duration = new Duration(DateTime.UtcNow - TimeSpan.FromDays(MaxDurationForCorrelationLookupInDays), DateTime.UtcNow);
var queryTracker = QueryStatTracker.CreateAndMarkQueryStart(ReaderConstants.CorrelatedEvents, eventInstanceId, queryParameters);
var events = new EventStoreReader()
.GetCorrelatedEvents(
eventInstanceGuid,
duration,
CancellationToken.None)
.GetAwaiter()
.GetResult()
.ToArray();
queryTracker.MarkQueryEnd(events.Count());
writer.Write(new JsonSerializerHelper().SerializeObject(events.Take(MaxRecordsInResult).OrderBy(item => item.TimeStamp)));
}
private static void HandlePartitionsEvents(IResultWriter writer, string partitionIdString, QueryParametersWrapper queryParameters)
{
if (!ValidateStartAndEndTime(queryParameters.StartTimeUtc, queryParameters.EndTimeUtc))
{
OnUnsupportedUri(writer);
return;
}
var partitionGuid = Guid.Empty;
if (!string.IsNullOrWhiteSpace(partitionIdString))
{
if (!Guid.TryParse(partitionIdString, out partitionGuid))
{
EventStoreLogger.Logger.LogWarning("HandlePartitionsEvents:: Partition ID : {0} Is not a valid Guid", partitionIdString);
OnUnsupportedUri(writer);
return;
}
}
var queryTracker = QueryStatTracker.CreateAndMarkQueryStart(ReaderConstants.Partitions, partitionIdString, queryParameters);
var events = new EventStoreReader()
.GetPartitionEventsAsync(
partitionGuid,
new Duration(
queryParameters.StartTimeUtc,
queryParameters.EndTimeUtc),
ConvertUserEventTypesToUnderlyingTypes(
EntityType.Partition,
queryParameters.EventsTypesFilter),
CancellationToken.None)
.GetAwaiter()
.GetResult()
.ToArray();
queryTracker.MarkQueryEnd(events.Count());
writer.Write(new JsonSerializerHelper().SerializeObject(events.Take(MaxRecordsInResult).OrderBy(item => item.TimeStamp)));
}
public SqliteDb(string db = null, bool useExistingDb = false, IResultWriter w = null, string delimiter = "\t", string dbLogPath = "")
{
try
{
if (db != null && useExistingDb == false)
{
File.Delete(db);
}
_dbname = db;
Delimiter = delimiter;
writer = w;
DbLogPath = dbLogPath;
if (!string.IsNullOrWhiteSpace(DbLogPath))
{
File.Delete(DbLogPath);
}
}
catch (Exception e)
{
writer = w;
writer.WriteError(e.Message);
}
}
private static void HandleClusterEvents(IResultWriter writer, QueryParametersWrapper queryParameters)
{
if (!ValidateStartAndEndTime(queryParameters.StartTimeUtc, queryParameters.EndTimeUtc))
{
OnUnsupportedUri(writer);
return;
}
var events = new EventStoreReader()
.GetClusterEventsAsync(
new Duration(
queryParameters.StartTimeUtc,
queryParameters.EndTimeUtc),
ConvertUserEventTypesToUnderlyingTypes(
EntityType.Cluster,
queryParameters.EventsTypesFilter),
CancellationToken.None)
.GetAwaiter()
.GetResult()
.ToArray();
writer.Write(new JsonSerializerHelper().SerializeObject(events.Take(MaxRecordsInResult).OrderBy(item => item.TimeStamp)));
}
public Program(SizePhotoOptions opts)
{
_opts = opts;
_pathHelper = _opts.GetPathHelper();
_category = new CategoryInfo {
Name = _opts.CategoryName,
Year = _opts.Year,
IsPrivate = _opts.IsPrivate
};
if(_opts.InsertMode)
{
_writer = new SqlInsertWriter(_opts.Outfile);
}
else if(_opts.UpdateMode)
{
_writer = new SqlUpdateWriter(_opts.Outfile);
}
else
{
_writer = new NoopWriter();
}
}
public void WriteResult(IResultWriter writer)
{
int maxStep = _percalationResult.Max();
var result = new Dictionary <int, double>(maxStep);
foreach (var percalationStep in _percalationResult)
{
var t = percalationStep;
while (t <= maxStep)
{
if (result.ContainsKey(t))
{
result[t]++;
}
else
{
result.Add(t, 1);
}
t++;
}
}
for (int i = 0; i <= maxStep; i++)
{
if (result.ContainsKey(i))
{
result[i] = (double)result[i] / (double)_calculationTask.CountOfIteration;
}
else
{
result.Add(i, 0);
}
}
writer.WriteDoubleValue(GrafCalculator.GetAvarageLinkCount(_netMap));
writer.WritePercalation(result);
}
private void OnSaveButtonClick(Object sender, EventArgs args)
{
IResultWriter affected = this.tvcContent.SelectedTab.Controls[0] as IResultWriter;
if (affected == null || !affected.HasResult)
{
return;
}
SaveFileDialog dialog = new SaveFileDialog()
{
Title = "Save Passwords",
Filter = "Text Files (*.txt)|*.txt|All Files (*.*)|*.*",
FileName = $"passwords-{DateTime.Now:yyyyMMddHHmmss}.txt",
RestoreDirectory = true
};
if (dialog.ShowDialog(this) != DialogResult.OK)
{
return;
}
try
{
using (Stream stream = dialog.OpenFile())
{
affected.WriteResult(stream, Encoding.UTF8);
}
}
catch (Exception exception)
{
MessageBox.Show(this,
$"Could not save file. See message below for more details. {Environment.NewLine.Repeat(2)}{exception.Message}",
"Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
public void ValidateFields(SchemaField schemaField, string tableName, IResultWriter writer)
{
var sqlbase = @"INSERT INTO " + tableName + "_Errors" +
@" SELECT {0}, '" + schemaField.Name + @"', {1},*
FROM " + tableName + @"
{2} LIMIT " + vOptions.QueryErrorLimit + ";";
string sql = "";
if (schemaField.Constraints.Required)
{
sql = String.Format(sqlbase, "'Required Field'", esettings.RequiredErrorLevel.SingleQuote(), " WHERE " + schemaField.Name + " = '' OR " + schemaField.Name + " IS NULL");
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Required Field check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Required Field check for " + schemaField.Name);
}
if (schemaField.DataType != DataType.@string &&
(schemaField.Constraints.Required || (esettings.ErrorOnUnrequiredWithBadDatatype && !schemaField.Constraints.Required)))
{
sql = GetSqlForDatatype(schemaField, sqlbase, esettings.DatatypeErrorLevel.SingleQuote());
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Datatype check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Datatype check for " + schemaField.Name);
}
if (!double.IsNaN(schemaField.Constraints.Maximum))
{
sql = String.Format(sqlbase, "'Maximum Value'", esettings.MaximumErrorLevel.SingleQuote(), "WHERE COMPARE(" + schemaField.Name + @", '>', " + schemaField.Constraints.Maximum + @", '" + schemaField.DataType.ToString() + "')");
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Maximum Value check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Maximum Value check for " + schemaField.Name);
}
if (!double.IsNaN(schemaField.Constraints.Minimum))
{
sql = String.Format(sqlbase, "'Minimum Value'", esettings.MinimumErrorLevel.SingleQuote(), "WHERE COMPARE(" + schemaField.Name + @", '<', " + schemaField.Constraints.Minimum + @", '" + schemaField.DataType.ToString() + "')");
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Minimum Value check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Minimum Value check for " + schemaField.Name);
}
if (schemaField.Constraints.MinLength > -1)
{
sql = String.Format(sqlbase, "'Minimum Length'", esettings.MinLengthErrorLevel.SingleQuote(), "WHERE LENGTH(" + schemaField.Name + @") < " + schemaField.Constraints.MinLength);
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Minimum Length check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Minimum Length check for " + schemaField.Name);
}
if (schemaField.Constraints.MaxLength > -1)
{
sql = String.Format(sqlbase, "'Maximum Length'", esettings.MaxLengthErrorLevel.SingleQuote(), "WHERE LENGTH(" + schemaField.Name + @") > " + schemaField.Constraints.MaxLength);
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Maximum Length check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Maximum Length check for " + schemaField.Name);
}
if ((schemaField.Constraints.Pattern != null) && (schemaField.Constraints.Pattern != String.Empty))
{
sql = String.Format(sqlbase, "'Invalid Value(pattern)'", esettings.PatternErrorLevel.SingleQuote(), "WHERE '" + schemaField.Constraints.Pattern + @"' NOT REGEXP " + schemaField.Name);
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Pattern check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Pattern check for " + schemaField.Name);
}
if (schemaField.Constraints.Enum != null && schemaField.Constraints.Enum.Length > 0)
{
var values = "(" + String.Join(",", schemaField.Constraints.Enum.ToList().Select(x => "'" + x + "'")) + ")";
sql = String.Format(sqlbase, "'Enum Valid Values'", esettings.MaxLengthErrorLevel.SingleQuote(), "WHERE " + schemaField.Name + @" NOT IN " + values);
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Enum Valid Values check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Enum Valid Values check for " + schemaField.Name);
}
if (schemaField.Constraints.Unique)
{
// Unique speed hack
sql = @"INSERT INTO " + tableName + "_Errors" +
@" SELECT 'Unique', '" + schemaField.Name + @"'," + esettings.UniqueErrorLevel.SingleQuote() + @" ,*
FROM " + tableName + @"
WHERE rowid IN (SELECT rowid FROM " + tableName + @" GROUP BY " + schemaField.Name + "HAVING COUNT(*) > 1;";
writer.WriteVerbose("[" + DateTime.Now + "] " + "Start Unique check for " + schemaField.Name);
SqliteStatus = db.ExecuteQuery(sql, Validate, ErrorCount);
writer.WriteVerbose("[" + DateTime.Now + "] " + "End Unique check for " + schemaField.Name);
}
}
protected abstract IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter);
public AsyncDataAdapter(IProvider provider, IReadOnlyCollection <AsyncDataAdapterCommand> commands, int maxRecords, int rowBlockSize, IResultWriter resultWriter,
Action <IAsyncDataAdapter, Exception> endFill, Action <IAsyncDataAdapter> writeEnd)
{
_provider = provider;
_commands = commands;
_maxRecords = maxRecords;
_rowBlockSize = rowBlockSize;
_resultWriter = resultWriter;
_endFill = endFill;
_writeEnd = writeEnd;
}
public static int Run(ConfigFile c, string cmd = "", string filename = "")
{
configFile = c;
if (!String.IsNullOrWhiteSpace(cmd))
{
try
{
Validate = new PlainTextResultWriter(Console.Out, Console.Out, Console.Out, Console.Out, configFile.Validate.Outputs.StandardOutputDelimiter);
string sql = cmd;
db = new SqliteDb(configFile.Db.DbName, true, null, configFile.Extract.Delimiter, configFile.Db.LogFile).Init();
SqliteStatus = db.ExecuteQuery(sql, Validate, "Command line query");
}
catch (Exception e)
{
Console.WriteLine("Something went wrong trying to read the input command. Try again... " + e.Message);
}
}
else if (!String.IsNullOrWhiteSpace(filename))
{
try
{
Validate = new PlainTextResultWriter(Console.Out, Console.Out, Console.Out, Console.Out, configFile.Validate.Outputs.StandardOutputDelimiter);
string sql = GetSqlContents(filename);
db = new SqliteDb(configFile.Db.DbName, true, null, configFile.Extract.Delimiter, configFile.Db.LogFile).Init();
SqliteStatus = db.ExecuteQuery(sql, Validate, "Command line file execution");
}
catch (Exception e)
{
Console.WriteLine("Something went wrong trying to read the input file. Try again... " + e.Message);
}
}
else
{
try
{
Validate = new PlainTextResultWriter(Console.Out, Console.Out, Console.Out, Console.Out);
Transform = new PlainTextResultWriter(Console.Out, Console.Out, Console.Out, Console.Out);
Load = new PlainTextResultWriter(Console.Out, Console.Out, Console.Out, Console.Out);
Validations validator = null;
SqliteStatus = 0;
string sql;
db = new SqliteDb(configFile.Db.DbName, configFile.Db.UseExistingDb, null, configFile.Extract.Delimiter, configFile.Db.LogFile).Init();
// NPS_TODO add error outputting for when these fail to load
if (configFile.Steps.Validate)
{
Validate = ConfigFileInit.InitValidateFromConfig(configFile);
}
if (configFile.Steps.Transform)
{
Transform = ConfigFileInit.InitTransformFromConfig(configFile);
}
if (configFile.Steps.Load)
{
Load = ConfigFileInit.InitLoadFromConfig(configFile, db);
}
DirectoryInfo schemaDirInfo = new DirectoryInfo(configFile.Extract.Schemas);
DirectoryInfo sourceFilesDirInfo = new DirectoryInfo(configFile.Extract.Source);
DirectoryInfo validationDirInfo = new DirectoryInfo(configFile.Validate.ValidationSource);
DirectoryInfo loadDirInfo = new DirectoryInfo(configFile.Load.LoadSource);
DirectoryInfo seedDirInfo = new DirectoryInfo(configFile.Extract.SeedData);
SchemaFile schemaFile = null;
SqliteModeler modeler = null;
Validate.BeginOutput("");
if (configFile.Steps.Extract)
{
//NPS_TODO: Add check to see if we need to do this on reuse db
sql = new SqliteModeler().CreateGeneralErrorWarningDdl();
SqliteStatus = db.ModifyDdlFromSqlString(sql);
Validate.WriteVerbose(SqliteStatus.ToString() + ":" + sql + "||" + db.LastError);
// load seed data
if (seedDirInfo.Exists)
{
foreach (var seedFile in seedDirInfo.GetFiles("*.sql"))
{
//NPS_TODO: Add check to see if we need to do this on reuse db
currentStep = SetCurrentStep("Reading seed data from " + seedFile.Name, Validate);
SqliteStatus = db.ExecuteQuery(File.ReadAllText(seedFile.FullName), Validate);
}
}
}
foreach (var file in schemaDirInfo.GetFiles("*.json"))
{
//NPS_TODO: See if Scheme has already been created
// create schemafile object
currentStep = SetCurrentStep("Reading schema file: " + file.Name, Validate);
schemaFile = JsonConvert.DeserializeObject <SchemaFile>(File.ReadAllText(file.FullName));
Validate.BeginContext(schemaFile.Name, Globals.ResultWriterDestination.stdOut);
if (configFile.Validate.Outputs.Warnings && (configFile.Validate.Outputs.StandardOutputConnectionString != configFile.Validate.Outputs.WarningsOutputConnectionString))
{
if (Validate.ResultMode == "delimited")
{
Validate.Write(schemaFile.Name, Globals.ResultWriterDestination.Warning);
}
else if (Validate.ResultMode == "json")
{
Validate.BeginContext(schemaFile.Name, Globals.ResultWriterDestination.Warning);
}
}
// create SQLiteModeler
currentStep = SetCurrentStep("Setting schema file: " + file.Name, Validate);
modeler = new SqliteModeler().SetSchemaFile(schemaFile);
// create SQL from schemafile
currentStep = SetCurrentStep("Creating DDL...", Validate);
sql = modeler.CreateDdl(true).Ddl;
// execute DDL in schemafile
currentStep = SetCurrentStep("Modifying SQLite DB with DDL...", Validate);
SqliteStatus = db.ModifyDdlFromSqlString(sql);
Validate.WriteVerbose(SqliteStatus.ToString() + ":" + sql);
// create SQL from schemafile
currentStep = SetCurrentStep("Creating DDL...", Validate);
sql = modeler.CreateErrorDdl().ErrorDdl;
// execute DDL in schemafile
currentStep = SetCurrentStep("Modifying SQLite DB with Error DDL...", Validate);
SqliteStatus = db.ModifyDdlFromSqlString(sql);
Validate.WriteVerbose(SqliteStatus.ToString() + ":" + sql);
// find linked flat file
var files = GetFullFilePath(sourceFilesDirInfo, schemaFile.Flatfile, Validate);
Flatfile flatfile = null;
if (configFile.Steps.Extract)
{
foreach (var f in files)
{
var reuseDbSql = "SELECT * FROM FileAudit WHERE FileName = '" + f + "';";
bool shouldReadFile = true;
if (configFile.Db.UseExistingDb && db.QueryHasResults(reuseDbSql))
{
shouldReadFile = false;
}
if (f != String.Empty && shouldReadFile)
{
//import flat file
currentStep = SetCurrentStep("Importing flatfile " + f + "...", Validate);
//NPS_TODO: setup File
flatfile = new Flatfile(f, schemaFile.Name, schemaFile.Delimiter.ToString(), "\"", true, null, schemaFile);
int linesRead = 0;
SqliteStatus = db.ImportDelimitedFile(flatfile, out linesRead, configFile, true);
// NPS_TODO: Make linenum optional in configfile
currentStep = SetCurrentStep("Finished reading flatfile " + f + "...", Validate);
var auditSql = "INSERT INTO FileAudit VALUES ('" + f + "', CURRENT_TIMESTAMP, " + schemaFile.Fields.Count + ", " + linesRead + ");";
SqliteStatus = db.ExecuteQuery(auditSql, Validate);
}
}
if (files.Count == 0)
{
SqliteStatus = db.ExecuteQuery("INSERT INTO GeneralErrors VALUES ('File Missing', 'None', '" + schemaFile.Name + "', 'Error', 'Failed to find file matching " + schemaFile.Flatfile + "');", Validate);
Validate.EndContext(Globals.ResultWriterDestination.stdOut);
continue; // no files, continue the loop so no validation happens
}
else
{
var metadataSql = "";
// NPS_TODO: Handle Derivations flag
// DERIVATIONS
foreach (var schemaField in flatfile.Schemafile.Fields.Where(x => x.ColumnType == ColumnType.Derived).Select(x => x))
{
var derivationSql = "UPDATE " + flatfile.Tablename + " SET " + schemaField.Name + " = " + schemaField.Derivation + ";";
SqliteStatus = db.ExecuteQuery(derivationSql, Validate);
}
foreach (var schemaField in schemaFile.Fields)
{
metadataSql = "INSERT INTO TableMetadata VALUES ('" + schemaFile.Name + "', '" + schemaField.Name + "', '" + schemaField.DataType + "');";
// finding numeric precision/scale for sql server
// with cte as (select length(b)-1 as precision, length(b) - instr(b, '.') as scale from foo) select case when
// max(precision) - min(scale) >= 38 then 38 else max(precision) end as precision, max(scale) from cte;
SqliteStatus = db.ExecuteQuery(metadataSql, Validate);
}
metadataSql = "INSERT INTO TableMetadata VALUES ('" + schemaFile.Name + "', 'LineNum', 'int');";
SqliteStatus = db.ExecuteQuery(metadataSql, Validate);
}
}
#region Validate
// file level validations
if (configFile.Steps.Validate)
{
validator = new Validations(configFile.Validate.SchemaErrorSettings, db, Validate, (code => SqliteStatus = code), configFile.Validate, schemaFile.Name);
currentStep = SetCurrentStep("Validating file", Validate);
foreach (var schemaField in schemaFile.Fields)
{
validator.ValidateFields(schemaField, schemaFile.Name, Validate);
}
if (schemaFile.SummarizeResults)
{
validator.PrintSummaryResults(schemaFile.Name, Globals.ResultWriterDestination.stdOut);
if (configFile.Validate.Outputs.Warnings)
{
validator.PrintSummaryResults(schemaFile.Name, Globals.ResultWriterDestination.Warning);
}
}
validator.PrintDetailResults(schemaFile.Name, Globals.ResultWriterDestination.stdOut);
if (configFile.Validate.Outputs.Warnings)
{
validator.PrintDetailResults(schemaFile.Name, Globals.ResultWriterDestination.Warning);
}
}
Validate.EndContext(Globals.ResultWriterDestination.stdOut);
if (Validate.ResultMode == "json" && configFile.Validate.Outputs.Warnings &&
(configFile.Validate.Outputs.StandardOutputConnectionString != configFile.Validate.Outputs.WarningsOutputConnectionString))
{
Validate.EndContext(Globals.ResultWriterDestination.Warning);
}
} // end for each flat file
//
// Custom validation checks
// Possibly cross file / multi table joins
//
if (configFile.Steps.Validate && !string.IsNullOrWhiteSpace(configFile.Validate.ValidationSource))
{
string ctx = "Custom Data Validation Checks";
// Perhaps we have no flatfiles but do have a db and custom validations - in this case validator would be null
if (validator == null)
{
validator = new Validations(configFile.Validate.SchemaErrorSettings, db, Validate, (code => SqliteStatus = code), configFile.Validate, "GeneralErrors");
}
Validate.BeginContext(ctx, Globals.ResultWriterDestination.stdOut);
if (configFile.Validate.Outputs.Warnings && (configFile.Validate.Outputs.StandardOutputConnectionString != configFile.Validate.Outputs.WarningsOutputConnectionString))
{
if (Validate.ResultMode == "delimited")
{
Validate.Write(ctx, Globals.ResultWriterDestination.Warning);
}
else if (Validate.ResultMode == "json")
{
Validate.BeginContext(ctx, Globals.ResultWriterDestination.Warning);
}
}
foreach (var validationFile in validationDirInfo.GetFiles("*.sql"))
{
currentStep = SetCurrentStep("Getting contents from: " + validationFile.Name, Validate);
validator.ValidateCustom(validationFile, configFile.Validate.QueryErrorLimit, configFile.Validate.Outputs.Warnings);
}
Validate.EndContext(Globals.ResultWriterDestination.stdOut);
if (Validate.ResultMode == "json" && configFile.Validate.Outputs.Warnings &&
(configFile.Validate.Outputs.StandardOutputConnectionString != configFile.Validate.Outputs.WarningsOutputConnectionString))
{
Validate.EndContext(Globals.ResultWriterDestination.Warning);
}
}
if (configFile.Steps.Validate)
{
validator.PrintGeneralIssues(Globals.ResultWriterDestination.stdOut);
if (configFile.Validate.Outputs.Warnings)
{
validator.PrintGeneralIssues(Globals.ResultWriterDestination.Warning);
}
}
Validate.EndOutput("");
#endregion Validate
Load.BeginOutput("");
if (configFile.Steps.Load)
{
foreach (var loadFile in loadDirInfo.GetFiles("*.sql"))
{
currentStep = SetCurrentStep("Getting contents from: " + loadFile.Name, Validate);
string context = String.Empty;
sql = GetSqlContents(loadFile.FullName);
context = loadFile.Name;
SqliteStatus = db.ExecuteQuery(sql, Load, context);
}
}
}
catch (Exception e)
{
Validate.WriteStd("ERROR on step: " + currentStep);
Validate.WriteError("[ERROR] " + DateTime.Now);
Validate.WriteError("[ERROR MSG] " + e.Message);
if (db != null && !string.IsNullOrWhiteSpace(db.LastError))
{
Validate.WriteError("[DB MSG] " + db.LastError);
}
Validate.WriteError(e.StackTrace);
return(SqliteStatus);
}
finally
{
db.Close();
Validate.Dispose();
}
}
return(SqliteStatus);
}
public DataSetResultWriter(Action <InfoMessage> addInfoMessage, bool showShemaTable)
{
_logResultWriter = new LogResultWriter(addInfoMessage);
_showShemaTable = showShemaTable;
}
public void Init()
{
_resultWriter = new ResultWriter(_crawlingStats);
}
public ConsoleResultWriter(IResultWriter resultWriter)
{
_resultWriter = resultWriter;
}
public DataFlowService(ILogger <DataFlowService> logger, IOptions <Configuration> configuration, IResultWriter resultWriter)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
_resultWriter = resultWriter;
}
public ConsoleResultWriter(IResultWriter resultWriter)
{
_resultWriter = resultWriter;
}
public JsonResultWriter(Action <InfoMessage> addInfoMessage)
{
_logResultWriter = new LogResultWriter(addInfoMessage);
}
public CommandProcessor(IResultWriter writer)
{
this.writer = writer;
}
static string SetCurrentStep(string desc, IResultWriter writer)
{
writer.WriteVerbose("[" + DateTime.Now + "] " + desc);
return(desc);
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter)
{
return new ParallelQueryMasterProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_spoolProcessingResponseDispatcher);
}
public ProgressResultWriter(EventSubscriptionBasedProjectionProcessingPhase phase,
IResultWriter resultWriter)
{
_phase = phase;
_resultWriter = resultWriter;
}
public ForJsonAutoResultWriter(Action <InfoMessage> addInfoMessage)
{
_addInfoMessage = addInfoMessage;
_logResultWriter = new LogResultWriter(addInfoMessage);
}
public MockCalculator(IResultWriter rw)
{
_resultWriter = rw;
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter)
{
var statePartitionSelector = CreateStatePartitionSelector();
var orderedPartitionProcessing = _sourceDefinition.ByStreams && _sourceDefinition.IsBiState;
return new EventProcessingProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_sourceDefinition.DefinesStateTransform,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
statePartitionSelector,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_sourceDefinition.IsBiState,
orderedPartitionProcessing: orderedPartitionProcessing);
}