Skip to content

Commit 22616a5

Browse files
dschwartz-ftadvisorydschwartz-ftadvisory
committed
feature: Refactor Python code generation to support namespace prefixes and externally set version #'s. Includes new unit tests and a reconfigured CDM build script
1 parent 1b79fa3 commit 22616a5

18 files changed

Lines changed: 844 additions & 226 deletions

‎python-test/cdm-tests/setup/build_cdm.sh‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,17 @@ CDM_SOURCE_PATH="$MY_PATH/../rosetta"
4646
PYTHON_TARGET_PATH=$PROJECT_ROOT_PATH/target/python-cdm
4747
PYTHON_SETUP_PATH="$MY_PATH/../../env-setup"
4848
JAR_PATH="$PROJECT_ROOT_PATH/target/python-0.0.0.main-SNAPSHOT.jar"
49-
CDM_PACKAGE_PREFIX="finos_cdm"
49+
CDM_PROJECT_NAME="finos_cdm"
50+
CDM_PREFIX="finos"
51+
CDM_VERSION="1.2.3"
5052
cd ${MY_PATH} || error
5153

5254

5355
source "$MY_PATH/../../common.sh" || { echo "Failed to source common.sh"; exit 1; }
5456

5557
# Parse command-line arguments
56-
# CDM_VERSION="master"
57-
CDM_VERSION="6.x.x"
58+
# CDM_BRANCH="master"
59+
CDM_BRANCH="6.x.x"
5860
SKIP_CDM=0
5961
for arg in "$@"; do
6062
case "$arg" in
@@ -66,21 +68,22 @@ for arg in "$@"; do
6668
;;
6769
*)
6870
# default any other argument to the CDM version
69-
CDM_VERSION="$arg"
71+
CDM_BRANCH="$arg"
7072
;;
7173
esac
7274
done
7375

7476
if [[ $SKIP_CDM -eq 0 ]]; then
75-
source $MY_PATH/get_cdm.sh "$CDM_VERSION"
77+
source $MY_PATH/get_cdm.sh "$CDM_BRANCH"
7678
else
7779
echo "Skipping get_cdm.sh as requested."
7880
fi
7981

8082
ensure_jar_exists "$PROJECT_ROOT_PATH" "$JAR_PATH"
8183

8284
echo "***** build CDM"
83-
java -cp "$JAR_PATH" com.regnosys.rosetta.generator.python.PythonCodeGeneratorCLI -s $CDM_SOURCE_PATH -t $PYTHON_TARGET_PATH -n $CDM_PACKAGE_PREFIX || error "Failed to generate CDM Python code"
85+
java -cp "$JAR_PATH" com.regnosys.rosetta.generator.python.PythonCodeGeneratorCLI -s $CDM_SOURCE_PATH -t $PYTHON_TARGET_PATH -n $CDM_PROJECT_NAME -v $CDM_VERSION -x $CDM_PREFIX|| error "Failed to generate CDM Python code"
86+
# java -cp "$JAR_PATH" com.regnosys.rosetta.generator.python.PythonCodeGeneratorCLI -s $CDM_SOURCE_PATH -t $PYTHON_TARGET_PATH || error "Failed to generate CDM Python code"
8487
JAVA_EXIT_CODE=$?
8588
if [[ $JAVA_EXIT_CODE -eq 1 ]]; then
8689
echo "Java program returned exit code 1. Stopping script."
@@ -100,11 +103,11 @@ python -m pip uninstall -y python-cdm 2>/dev/null
100103

101104
echo "***** build CDM Python package"
102105
cd $PYTHON_TARGET_PATH
103-
rm -f $CDM_PACKAGE_PREFIX-*.*.*-py3-none-any.whl
106+
rm -f *-*.*.*-py3-none-any.whl
104107
python -m pip wheel --no-deps --only-binary :all: . || error
105108

106109
echo "***** install CDM Python package"
107-
CDM_WHL=$(ls $CDM_PACKAGE_PREFIX-*.*.*-py3-none-any.whl 2>/dev/null | head -1)
110+
CDM_WHL=$(ls *-*.*.*-py3-none-any.whl 2>/dev/null | head -1)
108111
if [ -z "$CDM_WHL" ]; then
109112
echo "ERROR: cdm wheel was not produced. Stopping."
110113
error

‎python-test/cdm-tests/setup/get_cdm.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ echo "***** resetting the rosetta directory"
2626
rm -rf "${ROSETTA_DIR}"
2727
mkdir -p "${ROSETTA_DIR}/common-domain-model"
2828

29-
CDM_VERSION=${1:-"master"}
29+
CDM_BRANCH=${1:-"master"}
3030

31-
echo "***** pull CDM rosetta definitions ($CDM_VERSION)"
31+
echo "***** pull CDM rosetta definitions ($CDM_BRANCH)"
3232
TEMP_CDM="${MY_PATH}/../temp_cdm"
3333
rm -rf "${TEMP_CDM}"
3434
mkdir -p "${TEMP_CDM}"
@@ -43,7 +43,7 @@ rosetta-source/pom.xml
4343
EOF
4444

4545
git remote add origin https://github.com/finos/common-domain-model.git
46-
git pull --depth 1 origin $CDM_VERSION || error "git pull for CDM failed"
46+
git pull --depth 1 origin $CDM_BRANCH || error "git pull for CDM failed"
4747

4848
# Copy CDM files to the target 'common-domain-model' folder
4949
cp -r rosetta-source/src/main/rosetta/* "${ROSETTA_DIR}/common-domain-model/"

‎src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGenerator.java‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,30 @@ public void setProjectName(String projectName) {
136136
this.projectName = projectName;
137137
}
138138

139+
/**
140+
* Optional namespace prefix prepended to every generated namespace.
141+
* When set to "finos", "cdm.event.common" becomes "finos.cdm.event.common".
142+
*/
143+
private String namespacePrefix = null;
144+
145+
/**
146+
* Sets the namespace prefix applied to all generated namespaces.
147+
*
148+
* @param namespacePrefix the prefix (e.g. {@code "finos"}), or {@code null} for none
149+
*/
150+
public void setNamespacePrefix(String namespacePrefix) {
151+
this.namespacePrefix = namespacePrefix;
152+
}
153+
154+
/**
155+
* Returns the effective (prefix-aware) model name used as the context key
156+
* and subfolder path.
157+
*/
158+
private String effectiveModelName(RosettaModel model) {
159+
return com.regnosys.rosetta.generator.python.util.RuneToPythonMapper.applyPrefix(
160+
model.getName(), namespacePrefix);
161+
}
162+
139163
/**
140164
* The PythonCodeGenerator constructor.
141165
*/
@@ -153,15 +177,25 @@ public PythonCodeGenerator() {
153177

154178
// Phase 1: Accumulate all elements from all models into per-namespace contexts.
155179
for (RosettaModel model : models) {
156-
String nameSpace = PythonCodeGeneratorUtil.getNamespace(model);
157-
PythonCodeGeneratorContext context = contexts.computeIfAbsent(nameSpace, k -> new PythonCodeGeneratorContext());
180+
String effectiveName = effectiveModelName(model);
181+
String nameSpace = effectiveName.split("\\.")[0];
182+
PythonCodeGeneratorContext context = contexts.computeIfAbsent(nameSpace, k -> {
183+
PythonCodeGeneratorContext c = new PythonCodeGeneratorContext();
184+
c.setNamespacePrefix(namespacePrefix);
185+
return c;
186+
});
158187

159188
boolean hasContent = model.getElements().stream()
160189
.anyMatch(e -> e instanceof Data
161190
|| (e instanceof Function && !(e instanceof FunctionDispatch))
162191
|| e instanceof RosettaEnumeration);
163192
if (hasContent) {
164-
context.addSubfolder(model.getName());
193+
context.addSubfolder(effectiveName);
194+
}
195+
boolean hasFunctions = model.getElements().stream()
196+
.anyMatch(e -> e instanceof Function && !(e instanceof FunctionDispatch));
197+
if (hasFunctions) {
198+
context.addSubfolder(effectiveName + ".functions");
165199
}
166200
model.getElements().stream()
167201
.filter(Data.class::isInstance)
@@ -190,7 +224,7 @@ public PythonCodeGenerator() {
190224
@Override
191225
public Map<String, ? extends CharSequence> generate(Resource resource, RosettaModel model, String version) {
192226
Map<String, CharSequence> result = new HashMap<>();
193-
String nameSpace = PythonCodeGeneratorUtil.getNamespace(model);
227+
String nameSpace = effectiveModelName(model).split("\\.")[0];
194228
PythonCodeGeneratorContext context = contexts.get(nameSpace);
195229
if (context == null) {
196230
return result;
@@ -211,7 +245,7 @@ public PythonCodeGenerator() {
211245

212246
context.getClassObjects().putAll(pojoGenerator.generate(modelData, context));
213247
context.getFunctionObjects().putAll(functionGenerator.generate(modelFunctions, context));
214-
result.putAll(enumGenerator.generate(modelEnums));
248+
result.putAll(enumGenerator.generate(modelEnums, context));
215249

216250
return result;
217251
}

‎src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGeneratorCLI.java‎

Lines changed: 72 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import java.util.List;
1313
import java.util.Map;
1414
import java.util.Objects;
15+
import java.util.regex.Pattern;
1516
import java.util.stream.Collectors;
1617

1718
import org.apache.commons.cli.CommandLine;
@@ -68,6 +69,16 @@
6869
* process</li>
6970
* <li><b>-t, --tgt &lt;target-dir&gt;</b>: Target directory for generated
7071
* Python code (defaults to <code>./python</code> if not specified)</li>
72+
* <li><b>-v, --version &lt;version&gt;</b>: Version number for the generated
73+
* package, in <code>#.#.#</code> format (defaults to
74+
* <code>0.0.0</code>)</li>
75+
* <li><b>-n, --project-name &lt;projectName&gt;</b>: Override the
76+
* <code>pyproject.toml</code> project name (defaults to
77+
* <code>python-&lt;first-namespace-segment&gt;</code>)</li>
78+
* <li><b>-e, --allow-errors</b>: Continue generation even if validation
79+
* errors are present</li>
80+
* <li><b>-w, --fail-on-warnings</b>: Treat validation warnings as
81+
* errors</li>
7182
* <li><b>-h</b>: Print usage/help</li>
7283
* </ul>
7384
*
@@ -96,6 +107,16 @@ public class PythonCodeGeneratorCLI {
96107
*/
97108
private static final Logger LOGGER = LoggerFactory.getLogger(PythonCodeGeneratorCLI.class);
98109

110+
/**
111+
* Default version used when no {@code -v} option is provided.
112+
*/
113+
static final String DEFAULT_VERSION = "0.0.0";
114+
115+
/**
116+
* Regex that a version string must fully match: three dot-separated integers.
117+
*/
118+
private static final Pattern VALID_VERSION_PATTERN = Pattern.compile("\\d+\\.\\d+\\.\\d+");
119+
99120
/**
100121
* Public constructor for the CLI tool.
101122
*/
@@ -128,6 +149,12 @@ public final int run(String[] args) {
128149
Option projectNameOpt = Option.builder("n").longOpt("project-name").argName("projectName")
129150
.desc("Override the pyproject.toml project name (default: python-<first-namespace-segment>)")
130151
.hasArg().build();
152+
Option versionOpt = Option.builder("v").longOpt("version").argName("version")
153+
.desc("Package version in #.#.# format (default: " + DEFAULT_VERSION + ")")
154+
.hasArg().build();
155+
Option namespacePrefixOpt = Option.builder("x").longOpt("namespace-prefix").argName("namespacePrefix")
156+
.desc("Prefix to prepend to every generated namespace (e.g. finos)")
157+
.hasArg().build();
131158

132159
options.addOption(help);
133160
options.addOption(srcDirOpt);
@@ -136,6 +163,8 @@ public final int run(String[] args) {
136163
options.addOption(allowErrorsOpt);
137164
options.addOption(failOnWarningsOpt);
138165
options.addOption(projectNameOpt);
166+
options.addOption(versionOpt);
167+
options.addOption(namespacePrefixOpt);
139168

140169
CommandLineParser parser = new DefaultParser();
141170
try {
@@ -148,13 +177,24 @@ public final int run(String[] args) {
148177
boolean allowErrors = cmd.hasOption("e");
149178
boolean failOnWarnings = cmd.hasOption("w");
150179
String projectName = cmd.getOptionValue("n");
180+
String namespacePrefix = cmd.getOptionValue("x");
181+
182+
String version = DEFAULT_VERSION;
183+
if (cmd.hasOption("v")) {
184+
String rawVersion = cmd.getOptionValue("v");
185+
if (!VALID_VERSION_PATTERN.matcher(rawVersion).matches()) {
186+
LOGGER.error("Invalid version format '{}'. Expected #.#.# (e.g. 1.2.3).", rawVersion);
187+
return 1;
188+
}
189+
version = rawVersion;
190+
}
151191

152192
if (cmd.hasOption("s")) {
153193
String srcDir = cmd.getOptionValue("s");
154-
return translateFromSourceDir(srcDir, tgtDir, allowErrors, failOnWarnings, projectName);
194+
return translateFromSourceDir(srcDir, tgtDir, allowErrors, failOnWarnings, projectName, version, namespacePrefix);
155195
} else if (cmd.hasOption("f")) {
156196
String srcFile = cmd.getOptionValue("f");
157-
return translateFromSourceFile(srcFile, tgtDir, allowErrors, failOnWarnings, projectName);
197+
return translateFromSourceFile(srcFile, tgtDir, allowErrors, failOnWarnings, projectName, version, namespacePrefix);
158198
} else {
159199
LOGGER.error("Either a source directory (-s) or source file (-f) must be specified.");
160200
printUsage(options);
@@ -172,8 +212,15 @@ private static void printUsage(Options options) {
172212
formatter.printHelp("PythonCodeGeneratorCLI", options, true);
173213
}
174214

175-
private int translateFromSourceDir(String srcDir, String tgtDir, boolean allowErrors, boolean failOnWarnings,
176-
String projectName) {
215+
private int translateFromSourceDir(
216+
String srcDir,
217+
String tgtDir,
218+
boolean allowErrors,
219+
boolean failOnWarnings,
220+
String projectName,
221+
String version,
222+
String namespacePrefix
223+
) {
177224
// Find all .rosetta files in a directory
178225
Path srcDirPath = Paths.get(srcDir);
179226
if (!Files.exists(srcDirPath)) {
@@ -189,15 +236,22 @@ private int translateFromSourceDir(String srcDir, String tgtDir, boolean allowEr
189236
.filter(Files::isRegularFile)
190237
.filter(f -> f.getFileName().toString().endsWith(".rosetta"))
191238
.collect(Collectors.toList());
192-
return processRosettaFiles(rosettaFiles, tgtDir, allowErrors, failOnWarnings, projectName);
239+
return processRosettaFiles(rosettaFiles, tgtDir, allowErrors, failOnWarnings, projectName, version, namespacePrefix);
193240
} catch (IOException e) {
194241
LOGGER.error("Failed to process source directory: {}", srcDir, e);
195242
return 1;
196243
}
197244
}
198245

199-
private int translateFromSourceFile(String srcFile, String tgtDir, boolean allowErrors, boolean failOnWarnings,
200-
String projectName) {
246+
private int translateFromSourceFile(
247+
String srcFile,
248+
String tgtDir,
249+
boolean allowErrors,
250+
boolean failOnWarnings,
251+
String projectName,
252+
String version,
253+
String namespacePrefix
254+
) {
201255
Path srcFilePath = Paths.get(srcFile);
202256
if (!Files.exists(srcFilePath)) {
203257
LOGGER.error("Source file does not exist: {}", srcFile);
@@ -212,12 +266,19 @@ private int translateFromSourceFile(String srcFile, String tgtDir, boolean allow
212266
return 1;
213267
}
214268
List<Path> rosettaFiles = List.of(srcFilePath);
215-
return processRosettaFiles(rosettaFiles, tgtDir, allowErrors, failOnWarnings, projectName);
269+
return processRosettaFiles(rosettaFiles, tgtDir, allowErrors, failOnWarnings, projectName, version, namespacePrefix);
216270
}
217271

218272
// Common processing function
219-
private int processRosettaFiles(List<Path> rosettaFiles, String tgtDir, boolean allowErrors, boolean failOnWarnings,
220-
String projectName) {
273+
private int processRosettaFiles(
274+
List<Path> rosettaFiles,
275+
String tgtDir,
276+
boolean allowErrors,
277+
boolean failOnWarnings,
278+
String projectName,
279+
String version,
280+
String namespacePrefix
281+
) {
221282
LOGGER.info("Processing {} .rosetta files, writing to: {}", rosettaFiles.size(), tgtDir);
222283

223284
if (rosettaFiles.isEmpty()) {
@@ -237,14 +298,14 @@ private int processRosettaFiles(List<Path> rosettaFiles, String tgtDir, boolean
237298

238299
PythonCodeGenerator pythonCodeGenerator = injector.getInstance(PythonCodeGenerator.class);
239300
pythonCodeGenerator.setProjectName(projectName);
301+
pythonCodeGenerator.setNamespacePrefix(namespacePrefix);
240302
PythonModelLoader modelLoader = injector.getInstance(PythonModelLoader.class);
241303

242304
List<RosettaModel> models = modelLoader.getRosettaModels(resources);
243305
if (models.isEmpty()) {
244306
LOGGER.error("No valid Rosetta models found.");
245307
return 1;
246308
}
247-
String version = models.getFirst().getVersion();
248309

249310
LOGGER.info("Processing {} models, version: {}", models.size(), version);
250311

‎src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGeneratorContext.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,12 @@
1212
import org.jgrapht.graph.DefaultDirectedGraph;
1313
import org.jgrapht.graph.DefaultEdge;
1414

15+
import com.regnosys.rosetta.generator.python.util.RuneToPythonMapper;
1516
import com.regnosys.rosetta.rosetta.RosettaEnumeration;
17+
import com.regnosys.rosetta.rosetta.RosettaNamed;
1618
import com.regnosys.rosetta.rosetta.simple.Data;
1719
import com.regnosys.rosetta.rosetta.simple.Function;
20+
import com.regnosys.rosetta.types.RType;
1821

1922
public final class PythonCodeGeneratorContext {
2023
/**
@@ -87,6 +90,10 @@ public final class PythonCodeGeneratorContext {
8790
* partitioning and reused during DAG processing.
8891
*/
8992
private List<Set<String>> sccs = null;
93+
/**
94+
* The namespace prefix to prepend to all generated namespaces (e.g. "finos"), or null.
95+
*/
96+
private String namespacePrefix = null;
9097

9198
public PythonCodeGeneratorContext() {
9299
this.subfolders = new LinkedHashSet<>();
@@ -221,4 +228,32 @@ public List<Set<String>> getSccs() {
221228
public void setSccs(List<Set<String>> sccs) {
222229
this.sccs = sccs;
223230
}
231+
232+
public String getNamespacePrefix() {
233+
return namespacePrefix;
234+
}
235+
236+
public void setNamespacePrefix(String namespacePrefix) {
237+
this.namespacePrefix = namespacePrefix;
238+
}
239+
240+
public String getFullyQualifiedName(RosettaNamed rn) {
241+
return RuneToPythonMapper.getFullyQualifiedName(rn, namespacePrefix);
242+
}
243+
244+
public String getBundleObjectName(RosettaNamed rn) {
245+
return RuneToPythonMapper.getBundleObjectName(rn, namespacePrefix);
246+
}
247+
248+
public String getBundleObjectName(RosettaNamed rn, boolean useQuotes) {
249+
return RuneToPythonMapper.getBundleObjectName(rn, useQuotes, namespacePrefix);
250+
}
251+
252+
public String applyPrefix(String name) {
253+
return RuneToPythonMapper.applyPrefix(name, namespacePrefix);
254+
}
255+
256+
public String toPythonType(RType rt) {
257+
return RuneToPythonMapper.toPythonType(rt, namespacePrefix);
258+
}
224259
}

0 commit comments

Comments
 (0)