public SW4STM32ProjectParser(string reportDir, MCU[] supportedMCUs)
{
_Report = new BSPReportWriter(reportDir, "ParseReport.txt");
_SupportedMCUs = supportedMCUs;
foreach (var mcu in _SupportedMCUs)
{
_SupportedMCUNames.Add(mcu.ID);
}
}
public MemoryLayoutAndSubstitutionRules ToMemoryLayout(BSPReportWriter report)
{
var layout = new MemoryLayout {
DeviceName = Name, Memories = Memories.Select(m => m.ToMemoryDefinition()).ToList()
};
const string FLASHMemoryName = "FLASH";
const string FLASHMemoryName2 = "FLASH1";
Memory ram;
Dictionary <string, string> memorySubstitutionRulesForRAMMode = null;
if (MCU.Name.StartsWith("STM32H7") && MCU.Name.EndsWith("M4"))
{
ram = layout.TryLocateAndMarkPrimaryMemory(MemoryType.RAM, MemoryLocationRule.ByAddress(0x30000000), MemoryLocationRule.ByName("RAM_D2"));
}
else if (MCU.Name.StartsWith("STM32MP1"))
{
ram = layout.TryLocateAndMarkPrimaryMemory(MemoryType.RAM, MemoryLocationRule.ByName("RAM1"));
}
else
{
ram = layout.TryLocateAndMarkPrimaryMemory(MemoryType.RAM, MemoryLocationRule.ByAddress(0x20000000));
}
if (MCU.Name.StartsWith("STM32H7") && !MCU.Name.EndsWith("M4"))
{
//STM32H7 system file expects the ISR to be located at address 0x24000000 (D1_AXISRAM_BASE) and not at 0x20000000.
var mainMemoryForRAMMode = layout.TryLocateMemory(MemoryType.RAM, MemoryLocationRule.ByAddress(0x24000000), MemoryLocationRule.ByName("RAM_D2")) ?? throw new Exception("Failed to locate main memory for RAM mode");
memorySubstitutionRulesForRAMMode = new Dictionary <string, string> {
{ FLASHMemoryName, mainMemoryForRAMMode.Name },
{ FLASHMemoryName2, mainMemoryForRAMMode.Name },
{ ram.Name, mainMemoryForRAMMode.Name },
};
}
if (ram == null)
{
report.ReportMergeableError("Could not locate primary RAM for the MCU(s)", MCU.Name, true);
}
if (layout.TryLocateAndMarkPrimaryMemory(MemoryType.FLASH, MemoryLocationRule.ByName(FLASHMemoryName, FLASHMemoryName2)) == null)
{
throw new Exception("No FLASH found");
}
return(new MemoryLayoutAndSubstitutionRules(layout, memorySubstitutionRulesForRAMMode));
}
public void BuildConfigurationFixDatabase(BSPReportWriter reportWriter)
{
ConfigurationFixDatabase result = new ConfigurationFixDatabase
{
ConfigurationTable = _ReverseConditionTable.ConditionTable,
Frameworks = _ReverseConditionTable.Frameworks,
SourceFiles = _ReverseConditionTable.FileTable,
IncludeDirectories = _ReverseConditionTable.IncludeDirectoryTable,
};
var file = Path.Combine(_BSP.Directory, ConfigurationFixDatabase.FileName);
if (File.Exists(file))
{
result.ConfigurationFileEntries = XmlTools.LoadObject <ConfigurationFixDatabase>(file).ConfigurationFileEntries;
}
for (int i = 0; i < _ReverseConditionTable.IncludeDirectoryTable.Count; i++)
{
string physicalDir = GetFullPath(_ReverseConditionTable.IncludeDirectoryTable[i].ObjectName);
if (physicalDir.Contains("$$"))
{
continue;
}
var headers = Directory.GetFiles(physicalDir, "*.h", SearchOption.AllDirectories);
foreach (var hdr in headers)
{
string relPath = hdr.Substring(physicalDir.Length).TrimStart('\\').Replace('\\', '/');
result.Headers.Add(new ConfigurationFixDatabase.SecondaryObjectEntry {
ObjectName = relPath, PrimaryObjectIndex = i
});
}
}
result.Symbols = ComputeSymbolToFileMap(reportWriter);
XmlTools.SaveObject(result, file);
}
private List <ConfigurationFixDatabase.SecondaryObjectEntry> ComputeSymbolToFileMap(BSPReportWriter reportWriter)
{
var result = new HashSet <ConfigurationFixDatabase.SecondaryObjectEntry>();
if (_ReverseConditionTable.ConfigurationFixSamples == null)
{
return(result.ToList());
}
Dictionary <string, bool> fileBuildStatus = new Dictionary <string, bool>();
foreach (var sample in _ReverseConditionTable.ConfigurationFixSamples)
{
var sampleDir = GetFullPath(sample.SamplePath);
LoadedBSP.LoadedSample sampleObj = new LoadedBSP.LoadedSample
{
BSP = _BSP,
Directory = sampleDir,
Sample = XmlTools.LoadObject <EmbeddedProjectSample>(Path.Combine(sampleDir, "sample.xml"))
};
var mcu = _BSP.MCUs.First(m => m.ExpandedMCU.ID == sample.MCUID);
List <int> queue = Enumerable.Range(0, _ReverseConditionTable.FileTable.Count).ToList();
while (queue.Count > 0)
{
Console.WriteLine($"Analyzing {sampleObj.Sample.Name} ({result.Count} symbols mapped, {queue.Count} files left)...");
List <int> rejects = new List <int>();
List <int> handledFiles = new List <int>();
ConstructedConfiguration cfg = new ConstructedConfiguration(_ReverseConditionTable);
foreach (var i in queue)
{
var file = _ReverseConditionTable.FileTable[i];
string ext = Path.GetExtension(file.ObjectName).ToLower();
if (ext != ".c" && ext != ".cpp")
{
continue;
}
if (cfg.TryMerge(file))
{
handledFiles.Add(i);
}
else
{
rejects.Add(i);
}
}
var buildResult = Program.TestSingleSample(sampleObj, mcu, _TestDirectory, cfg.ToSampleJobObject(), null, null, BSPValidationFlags.KeepDirectoryAfterSuccessfulTest | BSPValidationFlags.ContinuePastCompilationErrors);
BuildSymbolTableFromBuildResults(handledFiles, result, fileBuildStatus);
if (rejects.Count == queue.Count)
{
break;
}
queue = rejects;
}
}
foreach (var kv in fileBuildStatus)
{
if (!kv.Value)
{
reportWriter.ReportMergeableMessage(BSPReportWriter.MessageSeverity.Warning, "Could not obtain symbol list provided by the following files", kv.Key, false);
}
}
return(result.ToList());
}
public override Dictionary <string, string> InsertVendorSamplesIntoBSP(ConstructedVendorSampleDirectory dir, VendorSample[] sampleList, string bspDirectory, BSPReportWriter reportWriter)
{
var copiedFiles = base.InsertVendorSamplesIntoBSP(dir, sampleList, bspDirectory, reportWriter);
Regex rgDebugger = new Regex("#define[ \t]+CFG_DEBUGGER_SUPPORTED[ \t]+(0)$");
foreach (var kv in copiedFiles)
{
if (kv.Value.EndsWith("app_conf.h", StringComparison.InvariantCultureIgnoreCase))
{
var lines = File.ReadAllLines(kv.Value);
bool modified = false;
for (int i = 0; i < lines.Length; i++)
{
var m = rgDebugger.Match(lines[i]);
if (m.Success)
{
lines[i] = lines[i].Substring(0, m.Groups[1].Index) + "1";
modified = true;
}
}
if (modified)
{
File.WriteAllLines(kv.Value, lines);
}
}
}
return(copiedFiles);
}
public void Run(string[] args)
{
string SDKdir = null;
string specificSampleName = null;
RunMode mode = RunMode.Invalid;
foreach (var arg in args)
{
string singlePrefix = "/single:";
if (arg.StartsWith(singlePrefix, StringComparison.InvariantCultureIgnoreCase))
{
mode = RunMode.SingleSample;
specificSampleName = arg.Substring(singlePrefix.Length);
}
else if (arg.StartsWith("/"))
{
mode = Enum.GetValues(typeof(RunMode)).OfType <RunMode>().First(v => v.ToString().ToLower() == arg.Substring(1).ToLower());
}
else
{
SDKdir = FilterSDKDir(arg);
}
}
if (SDKdir == null || mode == RunMode.Invalid)
{
Console.WriteLine($"Usage: {Path.GetFileName(Assembly.GetEntryAssembly().Location)} <mode> <SW package directory>");
Console.WriteLine($"Modes:");
Console.WriteLine($" /incremental - Only retest/rebuild previously failed samples.");
Console.WriteLine($" This doesn't update the BSP archive.");
Console.WriteLine($" /release - Reuse cached definitions, retest all samples. Update BSP.");
Console.WriteLine($" /cleanRelease - Reparse/retest all samples. Update BSP.");
Console.WriteLine($" /updateErrors - Re-categorize errors based on KnownProblems.xml");
Console.WriteLine($" /single:<name> - Run all phases of just one sample.");
Console.WriteLine($"Press any key to continue...");
Console.ReadKey();
Environment.ExitCode = 1;
return;
}
if (mode == RunMode.Incremental)
{
Console.WriteLine("*********************** WARNING ************************");
Console.WriteLine("* Vendor sample parser is running in incremental mode. *");
Console.WriteLine("* Only retested samples will be saved to BSP! *");
Console.WriteLine("* Re-run in /release mode to build a releasable BSP. *");
Console.WriteLine("********************************************************");
}
if (mode == RunMode.UpdateErrors)
{
foreach (var rec in _Report.Records)
{
}
XmlTools.SaveObject(_Report, ReportFile);
return;
}
string archiveName = string.Format("{0}-{1}.vgdbxbsp", BSP.BSP.PackageID.Split('.').Last(), BSP.BSP.PackageVersion);
string archiveFilePath = Path.Combine(BSPDirectory, archiveName);
if (File.Exists(archiveFilePath))
{
File.Delete(archiveFilePath);
}
string sampleListFile = Path.Combine(CacheDirectory, "Samples.xml");
var sampleDir = BuildOrLoadSampleDirectoryAndUpdateReportForFailedSamples(sampleListFile, SDKdir, mode, specificSampleName);
Dictionary <string, string> encounteredIDs = new Dictionary <string, string>();
foreach (var vs in sampleDir.Samples)
{
if (encounteredIDs.TryGetValue(vs.InternalUniqueID, out var dir))
{
throw new Exception("Duplicate sample for " + vs.InternalUniqueID);
}
encounteredIDs[vs.InternalUniqueID] = vs.Path;
var rec = _Report.ProvideEntryForSample(vs.InternalUniqueID);
if (rec.LastSucceededPass < VendorSamplePass.InitialParse)
{
rec.LastSucceededPass = VendorSamplePass.InitialParse;
}
}
//We cache unadjusted sample definitions to allow tweaking the adjusting code without the need to reparse everything.
Console.WriteLine("Adjusting sample properties...");
foreach (var vs in sampleDir.Samples)
{
AdjustVendorSampleProperties(vs);
if (vs.Path == null)
{
throw new Exception("Missing sample path for " + vs.UserFriendlyName);
}
}
VendorSample[] pass1Queue, insertionQueue;
switch (mode)
{
case RunMode.Incremental:
pass1Queue = insertionQueue = sampleDir.Samples.Where(s => _Report.ShouldBuildIncrementally(s.InternalUniqueID, VendorSamplePass.InPlaceBuild)).ToArray();
break;
case RunMode.Release:
insertionQueue = sampleDir.Samples;
if (sampleDir.Samples.FirstOrDefault(s => s.AllDependencies != null) == null)
{
pass1Queue = sampleDir.Samples;
}
else
{
pass1Queue = new VendorSample[0];
}
break;
case RunMode.CleanRelease:
pass1Queue = insertionQueue = sampleDir.Samples;
break;
case RunMode.SingleSample:
pass1Queue = insertionQueue = sampleDir.Samples.Where(s => s.InternalUniqueID == specificSampleName).ToArray();
if (pass1Queue.Length == 0)
{
throw new Exception("No samples match " + specificSampleName);
}
break;
default:
throw new Exception("Invalid run mode");
}
if (pass1Queue.Length > 0)
{
//Test the raw VendorSamples in-place and store AllDependencies
TestVendorSamplesAndUpdateReportAndDependencies(pass1Queue, null, VendorSamplePass.InPlaceBuild, vs => _Report.HasSampleFailed(vs.InternalUniqueID), validationFlags: BSPValidationFlags.ResolveNameCollisions);
foreach (var vs in pass1Queue)
{
if (vs.Path == null)
{
throw new Exception("Missing sample path for " + vs.UserFriendlyName);
}
}
sampleDir.ToolchainDirectory = ToolchainDirectory;
sampleDir.BSPDirectory = Path.GetFullPath(BSPDirectory);
XmlTools.SaveObject(sampleDir, sampleListFile);
}
//Insert the samples into the generated BSP
using (var reportWriter = new BSPReportWriter(CacheDirectory, "RelocationReport.txt"))
{
var relocator = CreateRelocator(sampleDir);
var copiedFiles = relocator.InsertVendorSamplesIntoBSP(sampleDir, insertionQueue, BSPDirectory, reportWriter);
var bsp = XmlTools.LoadObject <BoardSupportPackage>(Path.Combine(BSPDirectory, LoadedBSP.PackageFileName));
bsp.VendorSampleDirectoryPath = VendorSampleDirectoryName;
bsp.VendorSampleCatalogName = VendorSampleCatalogName;
XmlTools.SaveObject(bsp, Path.Combine(BSPDirectory, LoadedBSP.PackageFileName));
var reverseConditionTableFile = Path.Combine(BSPDirectory, ReverseFileConditionBuilder.ReverseConditionListFileName + ".gz");
if (File.Exists(reverseConditionTableFile))
{
Console.WriteLine("Building configuration fix database...");
var testDir = Path.Combine(TestDirectory, BSP.BSP.PackageID, "PassZ_AutoFixTest");
var fixBuilder = new ConfigurationFixDatabaseBuilder(BSP, testDir, XmlTools.LoadObject <ReverseConditionTable>(reverseConditionTableFile));
fixBuilder.BuildConfigurationFixDatabase(reportWriter);
}
}
if (mode != RunMode.Incremental && mode != RunMode.SingleSample)
{
Console.WriteLine("Creating new BSP archive...");
string statFile = Path.ChangeExtension(archiveName, ".xml");
TarPacker.PackDirectoryToTGZ(BSPDirectory, archiveFilePath, fn => Path.GetExtension(fn).ToLower() != ".vgdbxbsp" && Path.GetFileName(fn) != statFile && !fn.Contains(ReverseFileConditionBuilder.ReverseConditionListFileName));
}
var vendorSampleListInBSP = Path.Combine(BSPDirectory, VendorSampleDirectoryName, "VendorSamples.xml");
// Finally verify that everything builds
var expandedSamples = XmlTools.LoadObject <VendorSampleDirectory>(vendorSampleListInBSP);
expandedSamples.Path = Path.GetFullPath(Path.Combine(BSPDirectory, VendorSampleDirectoryName));
var finalStats = TestVendorSamplesAndUpdateReportAndDependencies(expandedSamples.Samples, expandedSamples.Path, VendorSamplePass.RelocatedBuild);
XmlTools.SaveObject(_Report, ReportFile);
if (mode == RunMode.Incremental || mode == RunMode.SingleSample)
{
Console.WriteLine($"Deleting incomplete {vendorSampleListInBSP}...\n***Re-run in /release mode to produce a valid BSP.");
File.Delete(vendorSampleListInBSP); //Incremental mode only places the samples that are currently built.
}
if (finalStats.Failed == 0)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"All {finalStats.Total} tests passed during final pass.");
}
else
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($"{finalStats.Failed} out of {finalStats.Total} tests failed during final pass.");
}
Console.ResetColor();
Console.WriteLine("=============================================");
Console.WriteLine($"Overall statistics for v{BSP.BSP.PackageVersion}:");
List <KeyValuePair <string, string> > fields = new List <KeyValuePair <string, string> >();
fields.Add(new KeyValuePair <string, string>("Total samples discovered:", sampleDir.Samples.Length.ToString()));
int unparsableSamples = _Report.Records.Count(r => r.LastSucceededPass == VendorSamplePass.None);
int failedAtFirstBuild = _Report.Records.Count(r => r.LastSucceededPass == VendorSamplePass.InitialParse && r.BuildFailedExplicitly);
int failedAtSecondBuild = _Report.Records.Count(r => r.LastSucceededPass == VendorSamplePass.InPlaceBuild && r.BuildFailedExplicitly);
fields.Add(new KeyValuePair <string, string>("Failed during initial parse attempt:", $"{unparsableSamples}/{sampleDir.Samples.Length} ({unparsableSamples * 100.0 / sampleDir.Samples.Length:f1}%)"));
fields.Add(new KeyValuePair <string, string>("Failed during in-place build:", $"{failedAtFirstBuild}/{sampleDir.Samples.Length} ({failedAtFirstBuild * 100.0 / sampleDir.Samples.Length:f1}%)"));
fields.Add(new KeyValuePair <string, string>("Failed during relocated build:", $"{failedAtSecondBuild}/{sampleDir.Samples.Length} ({failedAtSecondBuild * 100.0 / sampleDir.Samples.Length:f1}%)"));
if (mode != RunMode.Incremental)
{
fields.Add(new KeyValuePair <string, string>("Inserted into BSP:", expandedSamples.Samples.Length.ToString()));
}
OutputKeyValueList(fields);
if (finalStats.Failed > 0 && mode != RunMode.Incremental)
{
throw new Exception("Some of the vendor samples have failed the final test. Fix this before releasing the BSP.");
}
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
}