static void Main(string[] args)
{
SourceProcessor sourceProcessor= new SourceProcessor();
int numargs = args.GetLength(0);
if (args.GetLength(0)<1)
{
Console.WriteLine("CustomXMLBuilder ERROR: the directory where the source files are wasn't provided");
return;
}
Console.WriteLine("Running CustomXMLBuilder " + args[0]);
string dirPath = args[0];
string xmlOutput= "";
//Parse CPP source files for class definition macros: CLASS_CONSTRUCTOR, CHILD_CLASS, EXTENDS, ...
List<string> sourceFiles = new List<string>(Directory.EnumerateFiles(dirPath, "*.cpp", SearchOption.AllDirectories));
foreach (var file in sourceFiles)
{
xmlOutput += sourceProcessor.processCPPFile(file);
}
xmlOutput= sourceProcessor.resolveInlineClassRefs(xmlOutput);
int numErrors= sourceProcessor.m_checker.checkClassReferences();
{
System.IO.StreamWriter outputFile = new System.IO.StreamWriter("../config/definitions.xml");
// encoding=\"utf-8\"
outputFile.Write("<?xml version=\"1.0\"?>\n<DEFINITIONS>\n");
outputFile.Write(xmlOutput);
outputFile.Write("\n</DEFINITIONS>");
outputFile.Close();
Console.WriteLine("Source code correctly parsed and XML configuration succesfully built. {0} Kbs of code read.", sourceProcessor.numCharsProcessed / 1000);
}
}
internal void AddSourceOperator <K, V>(string topic, string nameNode, ConsumedInternal <K, V> consumed)
{
if (!sourceOperators.ContainsKey(nameNode))
{
SourceProcessor <K, V> source = new SourceProcessor <K, V>(nameNode, topic, consumed.KeySerdes, consumed.ValueSerdes, consumed.TimestampExtractor);
sourceOperators.Add(nameNode, source);
}
else
{
throw new Exception("Source operator already exist !");
}
}
private Dictionary <TopicPartition, RecordQueue> GetPartitions()
{
var timestampEx = new FailOnInvalidTimestamp();
var partitions = new Dictionary <TopicPartition, RecordQueue>();
var sourceProcessor = new SourceProcessor <string, string>("source1", "test", serdes, serdes, timestampEx);
var sourceProcessor2 = new SourceProcessor <string, string>("source2", "test2", serdes, serdes, timestampEx);
var recordQueue = new RecordQueue("", "source_queue", timestampEx, topicPart1, sourceProcessor);
var recordQueue2 = new RecordQueue("", "source2_queue", timestampEx, topicPart2, sourceProcessor2);
partitions.Add(topicPart1, recordQueue);
partitions.Add(topicPart2, recordQueue2);
return(partitions);
}
public void QueueOneMessageTest()
{
var timestampEx = new FailOnInvalidTimestamp();
var serdes = new StringSerDes();
var sourceProcessor = new SourceProcessor <string, string>("source", "test", serdes, serdes, timestampEx);
var recordQueue = new RecordQueue("", "", timestampEx, new TopicPartition("test", 0), sourceProcessor);
int size = recordQueue.Queue(new ConsumeResult <byte[], byte[]>()
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize("key", new SerializationContext()),
Value = serdes.Serialize("test", new SerializationContext())
}
});
Assert.AreEqual(1, size);
}
public void Initialize()
{
streamMetricsRegistry
= new StreamMetricsRegistry(Guid.NewGuid().ToString(),
MetricsRecordingLevel.DEBUG);
config.ApplicationId = "test-stream-thread";
config.StateDir = Guid.NewGuid().ToString();
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
config.CommitIntervalMs = 1;
id = new TaskId {
Id = 0, Partition = 0
};
topicPartition = new TopicPartition("topic", 0);
sourceProcessor = new SourceProcessor <string, string>(
processorNodeName,
"topic",
new StringSerDes(),
new StringSerDes(),
new FailOnInvalidTimestamp());
sourceProcessor.AddNextProcessor(new KStreamPeekProcessor <string, string>((k, v) => { }, false));
context = new ProcessorContext(
UnassignedStreamTask.Create(id),
config,
new ProcessorStateManager(
id,
new List <TopicPartition> {
topicPartition
},
null,
new StoreChangelogReader(config, null, threadId, streamMetricsRegistry),
new MockOffsetCheckpointManager()
), streamMetricsRegistry);
sourceProcessor.Init(context);
}
public void QueueDequeueOneMessageTest()
{
var timestampEx = new FailOnInvalidTimestamp();
var serdes = new StringSerDes();
var sourceProcessor = new SourceProcessor <string, string>("source", "test", serdes, serdes, timestampEx);
var recordQueue = new RecordQueue("", "", timestampEx, new TopicPartition("test", 0), sourceProcessor);
recordQueue.Queue(new ConsumeResult <byte[], byte[]>()
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize("key", new SerializationContext()),
Value = serdes.Serialize("test", new SerializationContext())
}
});
var r = recordQueue.Poll();
Assert.IsNotNull(r);
Assert.AreEqual("key", serdes.Deserialize(r.Message.Key, new SerializationContext()));
Assert.AreEqual("test", serdes.Deserialize(r.Message.Value, new SerializationContext()));
Assert.AreEqual(0, recordQueue.Size);
Assert.IsTrue(recordQueue.IsEmpty);
}
public List <string> GenerateScriptFromSourceServer(ServerConnection sc, List <DatabaseObject> objectList)
{
var dtStart = DateTime.Now;
AsyncUpdateStatus updateStatus = GenScriptAsyncUpdateStatusHandler;
var args = new AsyncNotificationEventArgs(NotificationEventFunctionCode.GenerateScriptFromSQLServer, 0, "", CommonFunc.FormatString("Process started at {0} -- UTC -> {1} ... ", dtStart.ToString(CultureInfo.CurrentUICulture), dtStart.ToUniversalTime().ToString(CultureInfo.CurrentUICulture)) + Environment.NewLine, Color.DarkBlue);
StreamWriter swTSQL = null;
SqlSmoObject[] smoTriggers = null;
Object sender = System.Threading.Thread.CurrentThread;
var smoScriptOpts = SMOScriptOptions.CreateFromConfig();
updateStatus(args);
sc.Connect();
var ss = new Server(sc);
var db = ss.Databases[sc.DatabaseName];
var sdb = new ScriptDatabase();
sdb.Initialize(ss, db, updateStatus, smoScriptOpts, swTSQL);
args.DisplayText = "";
args.StatusMsg = "Sorting objects by dependency ...";
args.PercentComplete = 1;
updateStatus(args);
var sorted = GetSortedObjects(db, objectList);
var sp = new SourceProcessor();
var tempPathForBCPData = Path.Combine(Path.GetTempPath(), "/BCPData");
Directory.CreateDirectory(tempPathForBCPData);
sp.Initialize(sdb, smoScriptOpts, updateStatus, args, tempPathForBCPData);
//NOTE: This is what does the magic!
if (sp.Process(sorted, 30))
{
return(output);
}
if (!Regex.IsMatch(smoScriptOpts.ScriptTableAndOrData, smoScriptOpts.GetLocalizedStringValue("ScriptOptionsTableData"), RegexOptions.IgnoreCase))
{
if (sp.Process(DatabaseObjectsTypes.Triggers, smoTriggers, 95))
{
return(output);
}
}
var dtEnd = DateTime.Now;
var tsDuration = dtEnd.Subtract(dtStart);
var sHour = tsDuration.Minutes == 1 ? " hour, " : " hours, ";
var sMin = tsDuration.Minutes == 1 ? " minute and " : " minutes and ";
var sSecs = tsDuration.Seconds == 1 ? " second" : " seconds";
args.StatusMsg = "Done!";
args.DisplayColor = Color.DarkCyan;
if (smoScriptOpts.CheckCompatibility() == 1)
{
args.DisplayText = CommonFunc.FormatString(@"No analysis done on script.
Processing finished at {0} -- UTC -> {1}
Total processing time: {2}", dtEnd.ToString(CultureInfo.CurrentUICulture), dtEnd.ToUniversalTime().ToString(CultureInfo.CurrentUICulture), tsDuration.Hours + sHour + tsDuration.Minutes.ToString(CultureInfo.CurrentUICulture) + sMin + tsDuration.Seconds.ToString(CultureInfo.CurrentUICulture) + sSecs);
}
else
{
args.DisplayText = CommonFunc.FormatString(@"Analysis completed at {0} -- UTC -> {1}
Any issues discovered will be reported above.
Total processing time: {2}", dtEnd.ToString(CultureInfo.CurrentUICulture), dtEnd.ToUniversalTime().ToString(CultureInfo.CurrentUICulture), tsDuration.Hours + sHour + tsDuration.Minutes.ToString(CultureInfo.CurrentUICulture) + sMin + tsDuration.Seconds.ToString(CultureInfo.CurrentUICulture) + sSecs);
}
args.PercentComplete = 100;
updateStatus(args);
return(output);
}
public void GenerateScriptFromSourceServer(Server sourceServer, Database sourceDatabase, StreamWriter swTSQL, AsyncUpdateStatus updateStatus, string outputDir, ObjectSelector objectFilter)
{
_updateStatus = updateStatus;
_smoScriptOpts = SMOScriptOptions.CreateFromConfig();
_ObjectSelector = objectFilter;
//if (_smoScriptOpts.TargetServer == _smoScriptOpts.GetLocalizedStringValue("ServerType_AzureSQLDatabase"))
//{
// _ObjectSelector.Assemblies.Script = false;
// _ObjectSelector.SchemaCollections.Script = false;
//}
DateTime dtStart = DateTime.Now;
AsyncNotificationEventArgs eventArgs = new AsyncNotificationEventArgs(NotificationEventFunctionCode.GenerateScriptFromSQLServer, 0, "", CommonFunc.FormatString(Properties.Resources.MessageProcessStarted, dtStart.ToString(CultureInfo.CurrentUICulture), dtStart.ToUniversalTime().ToString(CultureInfo.CurrentUICulture)) + Environment.NewLine, Color.DarkBlue);
_updateStatus(eventArgs);
ScriptDatabase sdb = new ScriptDatabase();
sdb.Initialize(sourceServer, sourceDatabase, _updateStatus, _smoScriptOpts, swTSQL, true);
eventArgs.DisplayText = "";
eventArgs.StatusMsg = Properties.Resources.MessageSorting;
eventArgs.PercentComplete = 1;
_updateStatus(eventArgs);
// Tables, Views, Stored Procedures, and Triggers can all have dependencies. GetSortedObjects returns
// these objects in dependency order.
SqlSmoObject[] smoAssemblies = GetAssemblies(sourceDatabase);
SqlSmoObject[] smoPartitionFunctions = GetPartitionFunctions(sourceDatabase);
SqlSmoObject[] smoPartitionSchemes = GetPartitionSchemes(sourceDatabase);
SqlSmoObject[] smoTriggers = GetTriggers(sourceServer, sourceDatabase);
SqlSmoObject[] smoRoles = GetRoles(sourceDatabase);
SqlSmoObject[] smoSchemas = GetSchemas(sourceDatabase);
SqlSmoObject[] smoSchemaCols = GetSchemaCollections(sourceServer, sourceDatabase);
SqlSmoObject[] smoUDTs = GetUDTs(sourceDatabase);
SqlSmoObject[] smoUDTTs = GetUDTTs(sourceDatabase);
SqlSmoObject[] smoSynonyms = GetSynonyms(sourceDatabase);
SqlSmoObject[] sorted = GetSortedObjects(sourceDatabase);
if (Regex.IsMatch(_smoScriptOpts.ScriptDropCreate, _smoScriptOpts.GetLocalizedStringValue("SOSDrop"), RegexOptions.IgnoreCase) ||
Regex.IsMatch(_smoScriptOpts.ScriptDropCreate, _smoScriptOpts.GetLocalizedStringValue("SOSDropCreate"), RegexOptions.IgnoreCase))
{
eventArgs.StatusMsg = Properties.Resources.MessageCreatingDropScripts;
eventArgs.PercentComplete = 2;
_updateStatus(eventArgs);
ScriptDrops(sorted, smoRoles, smoSchemas, smoSchemaCols, smoUDTs, smoUDTTs, smoAssemblies, smoSynonyms, smoPartitionFunctions, smoPartitionSchemes, sdb);
}
if (Regex.IsMatch(_smoScriptOpts.ScriptDropCreate, _smoScriptOpts.GetLocalizedStringValue("SOSCreate"), RegexOptions.IgnoreCase) ||
Regex.IsMatch(_smoScriptOpts.ScriptDropCreate, _smoScriptOpts.GetLocalizedStringValue("SOSDropCreate"), RegexOptions.IgnoreCase))
{
SourceProcessor sp = new SourceProcessor();
sp.Initialize(sdb, _smoScriptOpts, _updateStatus, eventArgs, outputDir);
// Roles, Schemas, XML Schema Collections and UDT have no dependencies. Thus we process one at a time.
if (!Regex.IsMatch(_smoScriptOpts.ScriptTableAndOrData, _smoScriptOpts.GetLocalizedStringValue("ScriptOptionsTableData"), RegexOptions.IgnoreCase))
{
if (sp.Process(DatabaseObjectsTypes.Assemblies, smoAssemblies, 3))
{
return;
}
if (sp.Process(DatabaseObjectsTypes.PartitionFunctions, smoPartitionFunctions, 6))
{
return;
}
if (sp.Process(DatabaseObjectsTypes.PartitionSchemes, smoPartitionSchemes, 9))
{
return;
}
if (sp.Process(DatabaseObjectsTypes.Roles, smoRoles, 12))
{
return;
}
if (sp.Process(DatabaseObjectsTypes.Schemas, smoSchemas, 15))
{
return;
}
if (sp.Process(DatabaseObjectsTypes.XMLSchemaCollections, smoSchemaCols, 20))
{
return;
}
if (sp.Process(DatabaseObjectsTypes.UserDefinedDataTypes, smoUDTs, 25))
{
return;
}
if (sp.Process(DatabaseObjectsTypes.UserDefinedTableTypes, smoUDTTs, 30))
{
return;
}
if (sp.Process(DatabaseObjectsTypes.Synonyms, smoSynonyms, 35))
{
return;
}
}
if (sp.Process(sorted, 40))
{
return;
}
if (!Regex.IsMatch(_smoScriptOpts.ScriptTableAndOrData, _smoScriptOpts.GetLocalizedStringValue("ScriptOptionsTableData"), RegexOptions.IgnoreCase))
{
if (sp.Process(DatabaseObjectsTypes.Triggers, smoTriggers, 95))
{
return;
}
}
}
if (swTSQL != null)
{
swTSQL.Flush();
swTSQL.Close();
}
DateTime dtEnd = DateTime.Now;
TimeSpan tsDuration = dtEnd.Subtract(dtStart);
string sHour = tsDuration.Minutes == 1 ? Properties.Resources.MessageHour : Properties.Resources.MessageHours;
string sMin = tsDuration.Minutes == 1 ? Properties.Resources.MessageMinute : Properties.Resources.MessageMinutes;
string sSecs = tsDuration.Seconds == 1 ? Properties.Resources.MessageSecond : Properties.Resources.MessageSeconds;
eventArgs.StatusMsg = Properties.Resources.Done;
eventArgs.DisplayColor = Color.DarkCyan;
if (_smoScriptOpts.CheckCompatibility() == 1)
{
eventArgs.DisplayText = CommonFunc.FormatString(Properties.Resources.MessageFinishedNoAnalysis, dtEnd.ToString(CultureInfo.CurrentUICulture), dtEnd.ToUniversalTime().ToString(CultureInfo.CurrentUICulture), tsDuration.Hours + sHour + tsDuration.Minutes.ToString(CultureInfo.CurrentUICulture) + sMin + tsDuration.Seconds.ToString(CultureInfo.CurrentUICulture) + sSecs);
}
else
{
eventArgs.DisplayText = CommonFunc.FormatString(Properties.Resources.MessageFinishedWithAnalysis, dtEnd.ToString(CultureInfo.CurrentUICulture), dtEnd.ToUniversalTime().ToString(CultureInfo.CurrentUICulture), tsDuration.Hours + sHour + tsDuration.Minutes.ToString(CultureInfo.CurrentUICulture) + sMin + tsDuration.Seconds.ToString(CultureInfo.CurrentUICulture) + sSecs);
}
eventArgs.PercentComplete = 100;
_updateStatus(eventArgs);
}
public static void Main(string[] args)
{
var sources = new[]
{
new XmltvSource(@"http://epg.iptvlive.org", 10),
new XmltvSource(@"http://epg.it999.ru/edem.xml.gz", 100),
new XmltvSource(@"http://programtv.ru/xmltv.xml.gz", 30)
};
string alphabet = "abcdefghijklmnopqrstuvwxyzабвгдеёжзийклмнопрстуфхцчшщъыьэюя";
var chars = new List <string>();
foreach (var c in alphabet)
{
chars.Add(c.ToString());
}
chars.Add(Other);
var dict = new Dictionary <string, IList <ChannelInfo> >();
foreach (var c in chars)
{
dict.Add(c, new List <ChannelInfo>());
}
var processor = new SourceProcessor();
foreach (var file in sources)
{
Log.Logger.Information($"The input file {file} is going to be processed");
Tv tv;
using (var stream = processor.Open(file))
{
XmlSerializer serializer = new XmlSerializer(typeof(Tv));
tv = (Tv)serializer.Deserialize(stream);
var names = tv.Channel.SelectMany(channel => channel.DisplayName
.Select(s => new ChannelInfo {
Name = s.Value, List = file.Path, Lang = s.Lang
}))
.ToList();
foreach (var info in names)
{
var key = info.Name.ToLowerInvariant()[0].ToString();
if (!chars.Contains(key))
{
dict[Other].Add(info);
}
else
{
dict[key].Add(info);
}
}
}
}
var sb = new StringBuilder();
foreach (var p in dict)
{
sb.AppendLine(p.Key);
foreach (var channelInfo in p.Value.OrderBy(info => info.Name))
{
sb.AppendLine($"{channelInfo.Name} -> {channelInfo.List}");
}
sb.AppendLine("----------------------------------");
}
File.WriteAllText("d:\\output.txt", sb.ToString());
Log.Logger.Information("Parsing has been completed succseefully");
}
