static void Main(string[] args) { SourceProcessor sourceProcessor= new SourceProcessor(); int numargs = args.GetLength(0); if (args.GetLength(0)<1) { Console.WriteLine("CustomXMLBuilder ERROR: the directory where the source files are wasn't provided"); return; } Console.WriteLine("Running CustomXMLBuilder " + args[0]); string dirPath = args[0]; string xmlOutput= ""; //Parse CPP source files for class definition macros: CLASS_CONSTRUCTOR, CHILD_CLASS, EXTENDS, ... List<string> sourceFiles = new List<string>(Directory.EnumerateFiles(dirPath, "*.cpp", SearchOption.AllDirectories)); foreach (var file in sourceFiles) { xmlOutput += sourceProcessor.processCPPFile(file); } xmlOutput= sourceProcessor.resolveInlineClassRefs(xmlOutput); int numErrors= sourceProcessor.m_checker.checkClassReferences(); { System.IO.StreamWriter outputFile = new System.IO.StreamWriter("../config/definitions.xml"); // encoding=\"utf-8\" outputFile.Write("<?xml version=\"1.0\"?>\n<DEFINITIONS>\n"); outputFile.Write(xmlOutput); outputFile.Write("\n</DEFINITIONS>"); outputFile.Close(); Console.WriteLine("Source code correctly parsed and XML configuration succesfully built. {0} Kbs of code read.", sourceProcessor.numCharsProcessed / 1000); } }
internal void AddSourceOperator <K, V>(string topic, string nameNode, ConsumedInternal <K, V> consumed) { if (!sourceOperators.ContainsKey(nameNode)) { SourceProcessor <K, V> source = new SourceProcessor <K, V>(nameNode, topic, consumed.KeySerdes, consumed.ValueSerdes, consumed.TimestampExtractor); sourceOperators.Add(nameNode, source); } else { throw new Exception("Source operator already exist !"); } }
private Dictionary <TopicPartition, RecordQueue> GetPartitions() { var timestampEx = new FailOnInvalidTimestamp(); var partitions = new Dictionary <TopicPartition, RecordQueue>(); var sourceProcessor = new SourceProcessor <string, string>("source1", "test", serdes, serdes, timestampEx); var sourceProcessor2 = new SourceProcessor <string, string>("source2", "test2", serdes, serdes, timestampEx); var recordQueue = new RecordQueue("", "source_queue", timestampEx, topicPart1, sourceProcessor); var recordQueue2 = new RecordQueue("", "source2_queue", timestampEx, topicPart2, sourceProcessor2); partitions.Add(topicPart1, recordQueue); partitions.Add(topicPart2, recordQueue2); return(partitions); }
public void QueueOneMessageTest() { var timestampEx = new FailOnInvalidTimestamp(); var serdes = new StringSerDes(); var sourceProcessor = new SourceProcessor <string, string>("source", "test", serdes, serdes, timestampEx); var recordQueue = new RecordQueue("", "", timestampEx, new TopicPartition("test", 0), sourceProcessor); int size = recordQueue.Queue(new ConsumeResult <byte[], byte[]>() { Message = new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize("test", new SerializationContext()) } }); Assert.AreEqual(1, size); }
public void Initialize() { streamMetricsRegistry = new StreamMetricsRegistry(Guid.NewGuid().ToString(), MetricsRecordingLevel.DEBUG); config.ApplicationId = "test-stream-thread"; config.StateDir = Guid.NewGuid().ToString(); config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 10; config.CommitIntervalMs = 1; id = new TaskId { Id = 0, Partition = 0 }; topicPartition = new TopicPartition("topic", 0); sourceProcessor = new SourceProcessor <string, string>( processorNodeName, "topic", new StringSerDes(), new StringSerDes(), new FailOnInvalidTimestamp()); sourceProcessor.AddNextProcessor(new KStreamPeekProcessor <string, string>((k, v) => { }, false)); context = new ProcessorContext( UnassignedStreamTask.Create(id), config, new ProcessorStateManager( id, new List <TopicPartition> { topicPartition }, null, new StoreChangelogReader(config, null, threadId, streamMetricsRegistry), new MockOffsetCheckpointManager() ), streamMetricsRegistry); sourceProcessor.Init(context); }
public void QueueDequeueOneMessageTest() { var timestampEx = new FailOnInvalidTimestamp(); var serdes = new StringSerDes(); var sourceProcessor = new SourceProcessor <string, string>("source", "test", serdes, serdes, timestampEx); var recordQueue = new RecordQueue("", "", timestampEx, new TopicPartition("test", 0), sourceProcessor); recordQueue.Queue(new ConsumeResult <byte[], byte[]>() { Message = new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize("test", new SerializationContext()) } }); var r = recordQueue.Poll(); Assert.IsNotNull(r); Assert.AreEqual("key", serdes.Deserialize(r.Message.Key, new SerializationContext())); Assert.AreEqual("test", serdes.Deserialize(r.Message.Value, new SerializationContext())); Assert.AreEqual(0, recordQueue.Size); Assert.IsTrue(recordQueue.IsEmpty); }
public List <string> GenerateScriptFromSourceServer(ServerConnection sc, List <DatabaseObject> objectList) { var dtStart = DateTime.Now; AsyncUpdateStatus updateStatus = GenScriptAsyncUpdateStatusHandler; var args = new AsyncNotificationEventArgs(NotificationEventFunctionCode.GenerateScriptFromSQLServer, 0, "", CommonFunc.FormatString("Process started at {0} -- UTC -> {1} ... ", dtStart.ToString(CultureInfo.CurrentUICulture), dtStart.ToUniversalTime().ToString(CultureInfo.CurrentUICulture)) + Environment.NewLine, Color.DarkBlue); StreamWriter swTSQL = null; SqlSmoObject[] smoTriggers = null; Object sender = System.Threading.Thread.CurrentThread; var smoScriptOpts = SMOScriptOptions.CreateFromConfig(); updateStatus(args); sc.Connect(); var ss = new Server(sc); var db = ss.Databases[sc.DatabaseName]; var sdb = new ScriptDatabase(); sdb.Initialize(ss, db, updateStatus, smoScriptOpts, swTSQL); args.DisplayText = ""; args.StatusMsg = "Sorting objects by dependency ..."; args.PercentComplete = 1; updateStatus(args); var sorted = GetSortedObjects(db, objectList); var sp = new SourceProcessor(); var tempPathForBCPData = Path.Combine(Path.GetTempPath(), "/BCPData"); Directory.CreateDirectory(tempPathForBCPData); sp.Initialize(sdb, smoScriptOpts, updateStatus, args, tempPathForBCPData); //NOTE: This is what does the magic! if (sp.Process(sorted, 30)) { return(output); } if (!Regex.IsMatch(smoScriptOpts.ScriptTableAndOrData, smoScriptOpts.GetLocalizedStringValue("ScriptOptionsTableData"), RegexOptions.IgnoreCase)) { if (sp.Process(DatabaseObjectsTypes.Triggers, smoTriggers, 95)) { return(output); } } var dtEnd = DateTime.Now; var tsDuration = dtEnd.Subtract(dtStart); var sHour = tsDuration.Minutes == 1 ? " hour, " : " hours, "; var sMin = tsDuration.Minutes == 1 ? " minute and " : " minutes and "; var sSecs = tsDuration.Seconds == 1 ? " second" : " seconds"; args.StatusMsg = "Done!"; args.DisplayColor = Color.DarkCyan; if (smoScriptOpts.CheckCompatibility() == 1) { args.DisplayText = CommonFunc.FormatString(@"No analysis done on script. Processing finished at {0} -- UTC -> {1} Total processing time: {2}", dtEnd.ToString(CultureInfo.CurrentUICulture), dtEnd.ToUniversalTime().ToString(CultureInfo.CurrentUICulture), tsDuration.Hours + sHour + tsDuration.Minutes.ToString(CultureInfo.CurrentUICulture) + sMin + tsDuration.Seconds.ToString(CultureInfo.CurrentUICulture) + sSecs); } else { args.DisplayText = CommonFunc.FormatString(@"Analysis completed at {0} -- UTC -> {1} Any issues discovered will be reported above. Total processing time: {2}", dtEnd.ToString(CultureInfo.CurrentUICulture), dtEnd.ToUniversalTime().ToString(CultureInfo.CurrentUICulture), tsDuration.Hours + sHour + tsDuration.Minutes.ToString(CultureInfo.CurrentUICulture) + sMin + tsDuration.Seconds.ToString(CultureInfo.CurrentUICulture) + sSecs); } args.PercentComplete = 100; updateStatus(args); return(output); }
public void GenerateScriptFromSourceServer(Server sourceServer, Database sourceDatabase, StreamWriter swTSQL, AsyncUpdateStatus updateStatus, string outputDir, ObjectSelector objectFilter) { _updateStatus = updateStatus; _smoScriptOpts = SMOScriptOptions.CreateFromConfig(); _ObjectSelector = objectFilter; //if (_smoScriptOpts.TargetServer == _smoScriptOpts.GetLocalizedStringValue("ServerType_AzureSQLDatabase")) //{ // _ObjectSelector.Assemblies.Script = false; // _ObjectSelector.SchemaCollections.Script = false; //} DateTime dtStart = DateTime.Now; AsyncNotificationEventArgs eventArgs = new AsyncNotificationEventArgs(NotificationEventFunctionCode.GenerateScriptFromSQLServer, 0, "", CommonFunc.FormatString(Properties.Resources.MessageProcessStarted, dtStart.ToString(CultureInfo.CurrentUICulture), dtStart.ToUniversalTime().ToString(CultureInfo.CurrentUICulture)) + Environment.NewLine, Color.DarkBlue); _updateStatus(eventArgs); ScriptDatabase sdb = new ScriptDatabase(); sdb.Initialize(sourceServer, sourceDatabase, _updateStatus, _smoScriptOpts, swTSQL, true); eventArgs.DisplayText = ""; eventArgs.StatusMsg = Properties.Resources.MessageSorting; eventArgs.PercentComplete = 1; _updateStatus(eventArgs); // Tables, Views, Stored Procedures, and Triggers can all have dependencies. GetSortedObjects returns // these objects in dependency order. SqlSmoObject[] smoAssemblies = GetAssemblies(sourceDatabase); SqlSmoObject[] smoPartitionFunctions = GetPartitionFunctions(sourceDatabase); SqlSmoObject[] smoPartitionSchemes = GetPartitionSchemes(sourceDatabase); SqlSmoObject[] smoTriggers = GetTriggers(sourceServer, sourceDatabase); SqlSmoObject[] smoRoles = GetRoles(sourceDatabase); SqlSmoObject[] smoSchemas = GetSchemas(sourceDatabase); SqlSmoObject[] smoSchemaCols = GetSchemaCollections(sourceServer, sourceDatabase); SqlSmoObject[] smoUDTs = GetUDTs(sourceDatabase); SqlSmoObject[] smoUDTTs = GetUDTTs(sourceDatabase); SqlSmoObject[] smoSynonyms = GetSynonyms(sourceDatabase); SqlSmoObject[] sorted = GetSortedObjects(sourceDatabase); if (Regex.IsMatch(_smoScriptOpts.ScriptDropCreate, _smoScriptOpts.GetLocalizedStringValue("SOSDrop"), RegexOptions.IgnoreCase) || Regex.IsMatch(_smoScriptOpts.ScriptDropCreate, _smoScriptOpts.GetLocalizedStringValue("SOSDropCreate"), RegexOptions.IgnoreCase)) { eventArgs.StatusMsg = Properties.Resources.MessageCreatingDropScripts; eventArgs.PercentComplete = 2; _updateStatus(eventArgs); ScriptDrops(sorted, smoRoles, smoSchemas, smoSchemaCols, smoUDTs, smoUDTTs, smoAssemblies, smoSynonyms, smoPartitionFunctions, smoPartitionSchemes, sdb); } if (Regex.IsMatch(_smoScriptOpts.ScriptDropCreate, _smoScriptOpts.GetLocalizedStringValue("SOSCreate"), RegexOptions.IgnoreCase) || Regex.IsMatch(_smoScriptOpts.ScriptDropCreate, _smoScriptOpts.GetLocalizedStringValue("SOSDropCreate"), RegexOptions.IgnoreCase)) { SourceProcessor sp = new SourceProcessor(); sp.Initialize(sdb, _smoScriptOpts, _updateStatus, eventArgs, outputDir); // Roles, Schemas, XML Schema Collections and UDT have no dependencies. Thus we process one at a time. if (!Regex.IsMatch(_smoScriptOpts.ScriptTableAndOrData, _smoScriptOpts.GetLocalizedStringValue("ScriptOptionsTableData"), RegexOptions.IgnoreCase)) { if (sp.Process(DatabaseObjectsTypes.Assemblies, smoAssemblies, 3)) { return; } if (sp.Process(DatabaseObjectsTypes.PartitionFunctions, smoPartitionFunctions, 6)) { return; } if (sp.Process(DatabaseObjectsTypes.PartitionSchemes, smoPartitionSchemes, 9)) { return; } if (sp.Process(DatabaseObjectsTypes.Roles, smoRoles, 12)) { return; } if (sp.Process(DatabaseObjectsTypes.Schemas, smoSchemas, 15)) { return; } if (sp.Process(DatabaseObjectsTypes.XMLSchemaCollections, smoSchemaCols, 20)) { return; } if (sp.Process(DatabaseObjectsTypes.UserDefinedDataTypes, smoUDTs, 25)) { return; } if (sp.Process(DatabaseObjectsTypes.UserDefinedTableTypes, smoUDTTs, 30)) { return; } if (sp.Process(DatabaseObjectsTypes.Synonyms, smoSynonyms, 35)) { return; } } if (sp.Process(sorted, 40)) { return; } if (!Regex.IsMatch(_smoScriptOpts.ScriptTableAndOrData, _smoScriptOpts.GetLocalizedStringValue("ScriptOptionsTableData"), RegexOptions.IgnoreCase)) { if (sp.Process(DatabaseObjectsTypes.Triggers, smoTriggers, 95)) { return; } } } if (swTSQL != null) { swTSQL.Flush(); swTSQL.Close(); } DateTime dtEnd = DateTime.Now; TimeSpan tsDuration = dtEnd.Subtract(dtStart); string sHour = tsDuration.Minutes == 1 ? Properties.Resources.MessageHour : Properties.Resources.MessageHours; string sMin = tsDuration.Minutes == 1 ? Properties.Resources.MessageMinute : Properties.Resources.MessageMinutes; string sSecs = tsDuration.Seconds == 1 ? Properties.Resources.MessageSecond : Properties.Resources.MessageSeconds; eventArgs.StatusMsg = Properties.Resources.Done; eventArgs.DisplayColor = Color.DarkCyan; if (_smoScriptOpts.CheckCompatibility() == 1) { eventArgs.DisplayText = CommonFunc.FormatString(Properties.Resources.MessageFinishedNoAnalysis, dtEnd.ToString(CultureInfo.CurrentUICulture), dtEnd.ToUniversalTime().ToString(CultureInfo.CurrentUICulture), tsDuration.Hours + sHour + tsDuration.Minutes.ToString(CultureInfo.CurrentUICulture) + sMin + tsDuration.Seconds.ToString(CultureInfo.CurrentUICulture) + sSecs); } else { eventArgs.DisplayText = CommonFunc.FormatString(Properties.Resources.MessageFinishedWithAnalysis, dtEnd.ToString(CultureInfo.CurrentUICulture), dtEnd.ToUniversalTime().ToString(CultureInfo.CurrentUICulture), tsDuration.Hours + sHour + tsDuration.Minutes.ToString(CultureInfo.CurrentUICulture) + sMin + tsDuration.Seconds.ToString(CultureInfo.CurrentUICulture) + sSecs); } eventArgs.PercentComplete = 100; _updateStatus(eventArgs); }
public static void Main(string[] args) { var sources = new[] { new XmltvSource(@"http://epg.iptvlive.org", 10), new XmltvSource(@"http://epg.it999.ru/edem.xml.gz", 100), new XmltvSource(@"http://programtv.ru/xmltv.xml.gz", 30) }; string alphabet = "abcdefghijklmnopqrstuvwxyzабвгдеёжзийклмнопрстуфхцчшщъыьэюя"; var chars = new List <string>(); foreach (var c in alphabet) { chars.Add(c.ToString()); } chars.Add(Other); var dict = new Dictionary <string, IList <ChannelInfo> >(); foreach (var c in chars) { dict.Add(c, new List <ChannelInfo>()); } var processor = new SourceProcessor(); foreach (var file in sources) { Log.Logger.Information($"The input file {file} is going to be processed"); Tv tv; using (var stream = processor.Open(file)) { XmlSerializer serializer = new XmlSerializer(typeof(Tv)); tv = (Tv)serializer.Deserialize(stream); var names = tv.Channel.SelectMany(channel => channel.DisplayName .Select(s => new ChannelInfo { Name = s.Value, List = file.Path, Lang = s.Lang })) .ToList(); foreach (var info in names) { var key = info.Name.ToLowerInvariant()[0].ToString(); if (!chars.Contains(key)) { dict[Other].Add(info); } else { dict[key].Add(info); } } } } var sb = new StringBuilder(); foreach (var p in dict) { sb.AppendLine(p.Key); foreach (var channelInfo in p.Value.OrderBy(info => info.Name)) { sb.AppendLine($"{channelInfo.Name} -> {channelInfo.List}"); } sb.AppendLine("----------------------------------"); } File.WriteAllText("d:\\output.txt", sb.ToString()); Log.Logger.Information("Parsing has been completed succseefully"); }