Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions tools/projmgr/include/ProjMgrMlops.h
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ class ProjMgrMlops {
* @return true if executed successfully
*/
bool CollectSettings(const CsolutionItem& csolution, MlopsType& mlops);
bool CollectContexts(const CsolutionItem& csolution, std::vector<std::string>& contexts) const;

private:
ProjMgrWorker* m_worker = nullptr;
Expand Down
14 changes: 14 additions & 0 deletions tools/projmgr/include/ProjMgrWorker.h
Original file line number Diff line number Diff line change
Expand Up @@ -961,6 +961,18 @@ class ProjMgrWorker {
*/
std::vector<std::string> GetSelectedContexts() const;

/**
* @brief update pack-relevant contexts from selected contexts and MLOps contexts
* @return true if executed successfully, false if context collection failed and the list was cleared
*/
bool UpdatePackContexts();

/**
* @brief get the list of pack-relevant contexts
* @return const reference to the list of pack-relevant context names
*/
const std::vector<std::string>& GetPackContexts() const;

/**
* @brief get the active target in the format <target-type>[@<target-set>]
* @return active target
Expand Down Expand Up @@ -1264,6 +1276,7 @@ class ProjMgrWorker {
m_processedContexts.clear();
m_contextErrMap.clear();
m_selectedContexts.clear();
m_packRelevantContexts.clear();
m_outputDir.clear();
m_selectedToolchain.clear();
m_rootDir.clear();
Expand Down Expand Up @@ -1297,6 +1310,7 @@ class ProjMgrWorker {
std::map<std::string, ContextItem> m_contexts;
std::map<std::string, std::set<std::string>> m_contextErrMap;
std::vector<std::string> m_selectedContexts;
std::vector<std::string> m_packRelevantContexts;
std::vector<ContextItem*> m_processedContexts;
std::string m_outputDir;
std::string m_packRoot;
Expand Down
12 changes: 7 additions & 5 deletions tools/projmgr/src/ProjMgr.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -576,14 +576,20 @@ bool ProjMgr::PopulateContexts(void) {
}

bool ProjMgr::GenerateYMLConfigurationFiles(bool previousResult) {
bool result = true;
MlopsType mlops;
if (m_parser.GetCsolution().mlops.enabled && !m_mlops.CollectSettings(m_parser.GetCsolution(), mlops)) {
result = false;
}

// Generate cbuild pack file
const bool isUsingContexts = m_contextSet || m_activeTargetSet.has_value() || m_context.size() != 0;
if (!m_emitter.GenerateCbuildPack(m_processedContexts, isUsingContexts, m_frozenPacks)) {
return false;
}

// Update the RTE files
bool result = UpdateRte();
result &= UpdateRte();

// Generate cbuild files
for (auto& contextItem : m_processedContexts) {
Expand Down Expand Up @@ -615,10 +621,6 @@ bool ProjMgr::GenerateYMLConfigurationFiles(bool previousResult) {

// Generate cbuild-mlops file
if (m_parser.GetCsolution().mlops.enabled) {
MlopsType mlops;
if (!m_mlops.CollectSettings(m_parser.GetCsolution(), mlops)) {
result = false;
}
if (!m_emitter.GenerateMlops(mlops)) {
result = false;
}
Expand Down
31 changes: 25 additions & 6 deletions tools/projmgr/src/ProjMgrCbuildIdx.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,16 @@ ProjMgrCbuildIdx::ProjMgrCbuildIdx(YAML::Node node,
node[YAML_CPROJECTS].push_back(cprojectNode);
}

vector<ContextItem*> additionalPackContexts;
map<string, ContextItem>* availableContexts = nullptr;
worker->GetContexts(availableContexts);
for (const auto& contextName : worker->GetPackContexts()) {
auto* packContext = &availableContexts->at(contextName);
if (find(processedContexts.begin(), processedContexts.end(), packContext) == processedContexts.end()) {
additionalPackContexts.push_back(packContext);
}
}

for (const auto& context : processedContexts) {
if (context) {
error_code ec;
Expand Down Expand Up @@ -163,17 +173,26 @@ ProjMgrCbuildIdx::ProjMgrCbuildIdx(YAML::Node node,
for (const auto& msg : msgVec) {
cbuildNode[YAML_MESSAGES][node].push_back(msg);
}
} else if (any_of(additionalPackContexts.begin(), additionalPackContexts.end(),
[&messageContextName = contextName](const ContextItem* packContext) { return packContext->name == messageContextName; })) {
for (const auto& msg : msgVec) {
cbuildNode[YAML_MESSAGES][node].push_back(contextName + ": " + msg);
}
}
}
}

vector<string> missingPacks;
for (const auto& packInfo : context->missingPacks) {
std::string missingPackName =
(packInfo.vendor.empty() ? "" : packInfo.vendor + "::") +
packInfo.name +
(packInfo.version.empty() ? "" : "@" + packInfo.version);
CollectionUtils::PushBackUniquely(missingPacks, missingPackName);
auto missingPackContexts = additionalPackContexts;
missingPackContexts.insert(missingPackContexts.begin(), context);
for (const auto* packContext : missingPackContexts) {
for (const auto& packInfo : packContext->missingPacks) {
const string missingPackName =
(packInfo.vendor.empty() ? "" : packInfo.vendor + "::") +
packInfo.name +
(packInfo.version.empty() ? "" : "@" + packInfo.version);
CollectionUtils::PushBackUniquely(missingPacks, missingPackName);
}
}

for (const auto& pack : missingPacks) {
Expand Down
11 changes: 10 additions & 1 deletion tools/projmgr/src/ProjMgrCbuildPack.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,16 @@ bool ProjMgrYamlEmitter::GenerateCbuildPack(const vector<ContextItem*> contexts,
}
// generate cbuild-pack.yml
const string& filename = m_parser->GetCsolution().directory + "/" + m_parser->GetCsolution().name + ".cbuild-pack.yml";
vector<ContextItem*> packContexts = contexts;
map<string, ContextItem>* availableContexts = nullptr;
m_worker->GetContexts(availableContexts);
for (const auto& contextName : m_worker->GetPackContexts()) {
auto* packContext = &availableContexts->at(contextName);
if (std::find(packContexts.begin(), packContexts.end(), packContext) == packContexts.end()) {
packContexts.push_back(packContext);
Comment thread
brondani marked this conversation as resolved.
Resolved
}
}
YAML::Node rootNode;
ProjMgrCbuildPack cbuildPack(rootNode[YAML_CBUILD_PACK], contexts, m_parser, keepExistingPackContent);
ProjMgrCbuildPack cbuildPack(rootNode[YAML_CBUILD_PACK], packContexts, m_parser, keepExistingPackContent);
return WriteFile(rootNode, filename, "", !cbuildPackFrozen);
}
23 changes: 23 additions & 0 deletions tools/projmgr/src/ProjMgrMlops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,29 @@ bool ProjMgrMlops::SetMlopsRunType(MlopsRunType& run, const string& targetType,
return true;
}

bool ProjMgrMlops::CollectContexts(const CsolutionItem& csolution, vector<string>& contexts) const {
if (!csolution.mlops.enabled) {
return true;
}
map<string, ContextItem>* availableContexts = nullptr;
m_worker->GetContexts(availableContexts);
for (const bool simulator : { false, true }) {
string targetType;
TargetSetItem targetSet;
if (!ResolveTargetSet(csolution, simulator ? csolution.mlops.simulator : csolution.mlops.hardware,
simulator, targetType, targetSet)) {
return false;
}
for (const auto& image : targetSet.images) {
const string contextName = (image.context.empty() ? csolution.name : image.context) + "+" + targetType;
if (availableContexts->find(contextName) != availableContexts->end()) {
CollectionUtils::PushBackUniquely(contexts, contextName);
}
}
}
return true;
}

bool ProjMgrMlops::CollectSettings(const CsolutionItem& csolution, MlopsType& mlops) {
mlops = {};

Expand Down
53 changes: 43 additions & 10 deletions tools/projmgr/src/ProjMgrWorker.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

#include "ProjMgrWorker.h"
#include "ProjMgrLogger.h"
#include "ProjMgrMlops.h"
#include "ProjMgrYamlEmitter.h"

#include "CrossPlatformUtils.h"
Expand Down Expand Up @@ -518,16 +519,32 @@ bool ProjMgrWorker::InitializeModel() {
return m_kernel->Init();
}

bool ProjMgrWorker::UpdatePackContexts() {
m_packRelevantContexts = m_selectedContexts;
if (m_parser && !ProjMgrMlops(this).CollectContexts(m_parser->GetCsolution(), m_packRelevantContexts)) {
m_packRelevantContexts.clear();
return false;
}
return true;
}

const vector<string>& ProjMgrWorker::GetPackContexts() const {
return m_packRelevantContexts;
}

bool ProjMgrWorker::CollectAllRequiredPdscFiles() {
// Check and collect requirements
if (m_selectedContexts.empty()) {
for (const auto& [context,_] : m_contexts) {
m_selectedContexts.push_back(context);
}
if (!UpdatePackContexts()) {
return false;
}
}
bool success = true;
m_contextErrMap.clear();
for (const auto& context : m_selectedContexts) {
for (const auto& context : m_packRelevantContexts) {
auto& contextItem = m_contexts.at(context);
if (!CollectRequiredPdscFiles(contextItem, m_packRoot)) {
success &= false;
Expand All @@ -542,7 +559,7 @@ bool ProjMgrWorker::LoadAllRelevantPacks() {
// Get required pdsc files
std::list<std::string> pdscFiles;
bool noPackRequirements = true;
for (const auto& context : m_selectedContexts) {
for (const auto& context : m_packRelevantContexts) {
Comment thread
brondani marked this conversation as resolved.
Resolved
auto& contextItem = m_contexts.at(context);
noPackRequirements &= contextItem.packRequirements.empty();
for (const auto& [pdscFile, pathVer] : contextItem.pdscFiles) {
Expand All @@ -566,7 +583,14 @@ bool ProjMgrWorker::LoadAllRelevantPacks() {
}
const set<string> requiredPdscFiles(pdscFiles.begin(), pdscFiles.end());
// Get installed packs
if (noPackRequirements || (m_loadPacksPolicy == LoadPacksPolicy::ALL) || (m_loadPacksPolicy == LoadPacksPolicy::LATEST)) {
// Additional MLOps requirements must not disable the default installed-pack fallback
// when the selected contexts have no pack requirements.
const bool useDefaultPackFallback = (m_loadPacksPolicy == LoadPacksPolicy::DEFAULT) &&
all_of(m_selectedContexts.begin(), m_selectedContexts.end(), [this](const string& context) {
return m_contexts.at(context).packRequirements.empty();
});
if (noPackRequirements || useDefaultPackFallback ||
(m_loadPacksPolicy == LoadPacksPolicy::ALL) || (m_loadPacksPolicy == LoadPacksPolicy::LATEST)) {
const bool latest = (m_loadPacksPolicy == LoadPacksPolicy::LATEST) || (m_loadPacksPolicy == LoadPacksPolicy::DEFAULT);
if (!m_kernel->GetEffectivePdscFiles(pdscFiles, latest)) {
ProjMgrLogger::Get().Error("parsing installed packs failed");
Expand Down Expand Up @@ -605,7 +629,9 @@ bool ProjMgrWorker::LoadPacks(ContextItem& context) {
}
bool deferredFail = false;
if (!CollectAllRequiredPdscFiles()) {
PrintContextErrors(context.name);
for (const auto& [contextName, errors] : m_contextErrMap) {
PrintContextErrors(contextName);
}
// defer return due to pdsc collection failure
deferredFail = true;
}
Expand Down Expand Up @@ -4229,7 +4255,7 @@ bool ProjMgrWorker::ListPacks(vector<string>&packs, bool bListMissingPacksOnly,
}
bool reqOk = true;
m_contextErrMap.clear();
for (const auto& selectedContext : m_selectedContexts) {
for (const auto& selectedContext : m_packRelevantContexts) {
ContextItem& context = m_contexts[selectedContext];
if(!CollectRequiredPdscFiles(context, m_packRoot)) {
// in case of explicit query for missing pack list,
Expand Down Expand Up @@ -4290,15 +4316,15 @@ bool ProjMgrWorker::ListPacks(vector<string>&packs, bool bListMissingPacksOnly,
}
// get available updates for locked packs
if (bLocked) {
for (const auto& selectedContext : m_selectedContexts) {
for (const auto& selectedContext : m_packRelevantContexts) {
ContextItem& context = m_contexts[selectedContext];
availablePackVersions.merge(context.availablePackVersions);
}
}
}

if (!m_contextErrMap.empty()) {
for (const auto& selectedContext : m_selectedContexts) {
for (const auto& selectedContext : m_packRelevantContexts) {
PrintContextErrors(selectedContext);
}
reqOk = false;
Expand Down Expand Up @@ -5290,6 +5316,7 @@ bool ProjMgrWorker::ProcessSequencesRelatives(ContextItem& context, BuildType& b
bool ProjMgrWorker::ParseContextSelection(
const vector<string>& contextSelection, const bool checkCbuildSet)
{
m_packRelevantContexts.clear();
vector<string> ymlOrderedContexts;
ListContexts(ymlOrderedContexts, RteUtils::EMPTY_STRING, true);

Expand Down Expand Up @@ -5380,7 +5407,10 @@ bool ProjMgrWorker::ParseContextSelection(
}

// Parse context layers
for (const auto& context : m_selectedContexts) {
if (!UpdatePackContexts()) {
return false;
}
for (const auto& context : m_packRelevantContexts) {
if (!ParseContextLayers(m_contexts[context])) {
return false;
}
Expand Down Expand Up @@ -5856,6 +5886,9 @@ bool ProjMgrWorker::ProcessGlobalGenerators(ContextItem* selectedContext, const
}
m_selectedContexts.push_back(context.name);
}
if (!UpdatePackContexts()) {
return false;
}
}
// parse layers and process selected contexts
bool deferredFail = false;
Expand Down Expand Up @@ -5914,7 +5947,7 @@ bool ProjMgrWorker::ProcessGlobalGenerators(ContextItem* selectedContext, const
projectType = type;
siblings = contexts;
m_selectedContexts = siblings;
return true;
return UpdatePackContexts();
}
}
return false;
Expand Down Expand Up @@ -6506,7 +6539,7 @@ void ProjMgrWorker::FormatResolvedPackIds() {
const auto& realPackId = loadedPack->GetPackageID();
realPackIds[RteUtils::ToLower(realPackId)] = realPackId;
}
for (auto& contextName : m_selectedContexts) {
for (const auto& contextName : m_packRelevantContexts) {
auto& contextItem = m_contexts[contextName];
for (auto& [_, resolvedPackIds] : contextItem.userInputToResolvedPackIdMap) {
map<string, PackageItem> formattedPackIds;
Expand Down
48 changes: 48 additions & 0 deletions tools/projmgr/test/data/MLOps/pack_contexts.csolution.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/Open-CMSIS-Pack/devtools/main/tools/projmgr/schemas/csolution.schema.json

solution:
compiler: AC6

packs:
- pack: ARM::RteTest_DFP
- pack: Missing::Simulator
for-context: +Simulator
- pack: Missing::Inactive
for-context: +Inactive

mlops:
description: ML model with context-specific pack requirements
model:
clayer: $AI-Layer$

target-types:
- type: Hardware
device: RteTest_ARMCM0_Dual
variables:
- AI-Layer: $SolutionDir()$/ai_layer/ai_layer.clayer.yml
target-set:
- set:
images:
- project-context: core0
- project-context: core1

- type: Simulator
device: RteTest_ARMCM0_Dual
variables:
- AI-Layer: $SolutionDir()$/ai_layer/ai_layer.clayer.yml
target-set:
- set: FVP-Test
debugger:
name: Arm-FVP
model: FVP_Corstone_SSE-320
config-file: fvp/fvp_config.txt
images:
- project-context: core0
- project-context: core1

- type: Inactive
device: RteTest_ARMCM0_Dual

projects:
- project: core0/core0.cproject.yml
- project: core1/core1.cproject.yml
Loading
Loading