From b246a88dea73534cf19668603539759e8acc73ec Mon Sep 17 00:00:00 2001 From: Felix Oesterle Date: Thu, 18 Sep 2025 10:34:33 +0200 Subject: [PATCH 1/2] chore(config): remove `.codeclimate.yml` and add new log helper scripts - Deleted `.codeclimate.yml` due to discontinued use in the project. - Added `logHelper_extractProcessAndTimes.sh` to extract process and timing information from log files. - Added `logHelper_extractProcessTimesAndParams.sh` to extract detailed simulation parameters and timing. --- .codeclimate.yml | 74 -------- avaframe/com1DFA/com1DFA.py | 1 + .../logHelper_extractProcessAndTimes.sh | 92 ++++++++++ .../logHelper_extractProcessTimesAndParams.sh | 164 ++++++++++++++++++ 4 files changed, 257 insertions(+), 74 deletions(-) delete mode 100644 .codeclimate.yml create mode 100755 tinyHelper/logHelper_extractProcessAndTimes.sh create mode 100755 tinyHelper/logHelper_extractProcessTimesAndParams.sh diff --git a/.codeclimate.yml b/.codeclimate.yml deleted file mode 100644 index 38caa8822..000000000 --- a/.codeclimate.yml +++ /dev/null @@ -1,74 +0,0 @@ -version: "2" -checks: - argument-count: - enabled: true - config: - threshold: 6 - complex-logic: - enabled: true - config: - threshold: 4 - file-lines: - enabled: true - config: - threshold: 300 - method-complexity: - enabled: true - config: - threshold: 5 - method-count: - enabled: true - config: - threshold: 20 - method-lines: - enabled: true - config: - threshold: 30 - nested-control-flow: - enabled: true - config: - threshold: 4 - return-statements: - enabled: true - config: - threshold: 4 - similar-code: - enabled: true - config: - threshold: #language-specific defaults. overrides affect all languages. - identical-code: - enabled: true - config: - threshold: #language-specific defaults. overrides affect all languages. -plugins: - pep8: - enabled: false -exclude_patterns: -- "config/" -- "db/" -- "dist/" -- "**/.*" -- "**/data/" -- "**/runScripts/" -- "**/com4FlowPy/" -- "**/tinyHelper/" -- "features/" -- "**/node_modules/" -- "script/" -- "**/spec/" -- "**/test/" -- "**/tests/" -- "Tests/" -- "docs/" -- "benchmarks/" -- "**/vendor/" -- "**/*_test.go" -- "**/*.d.ts" -- "**/*.cint" -- "**/*.yml" -- "**/*.ini" -- "**/*.json" -- "**/*.txt" -- "**/*.conf" -- "**/run*.py" -- ".github/" diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index f7f6aa3fd..9da4586a4 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -2975,6 +2975,7 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting [ relNameSim, simHash, + modName, defID, frictIndi or volIndi, row._asdict()["simTypeList"], diff --git a/tinyHelper/logHelper_extractProcessAndTimes.sh b/tinyHelper/logHelper_extractProcessAndTimes.sh new file mode 100755 index 000000000..40d9a501c --- /dev/null +++ b/tinyHelper/logHelper_extractProcessAndTimes.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# Script to extract and match "runs as process" lines with their corresponding "cpu time DFA" lines +# Usage: ./extractProcessAndTimes + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + echo "Example: $0 runAna4ProbAna_20250917_14h34m46s.log" + exit 1 +fi + +logFile="$1" + +if [ ! -f "$logFile" ]; then + echo "Error: File '$logFile' not found!" + exit 1 +fi + +echo "Extracting process and timing information from: $logFile" +echo "================================================================" +echo + +# Create temporary files to store the extracted data +processLines=$(mktemp) +timingLines=$(mktemp) + +# Extract all "runs as process" lines with line numbers +grep -n "runs as process" "$logFile" > "$processLines" + +# Extract all "cpu time DFA" lines with line numbers +grep -n "cpu time DFA" "$logFile" > "$timingLines" + +# Function to extract process ID from a line +getProcessId() { + echo "$1" | cut -d',' -f1 | cut -d':' -f2 +} + +# Function to extract simulation name from process line +getSimName() { + echo "$1" | sed 's/.*INFO - \(.*\) runs as process.*/\1/' +} + +# Function to extract CPU time from timing line +getCpuTime() { + echo "$1" | sed 's/.*cpu time DFA = \([0-9.]*\) s.*/\1/' +} + +# Counter for matched pairs +counter=1 + +echo "Matched Process and Timing Pairs:" +echo "==================================" + +# Read process lines and find matching timing lines +while IFS= read -r processLine; do + processId=$(getProcessId "$processLine") + simName=$(getSimName "$processLine") + processLineNum=$(echo "$processLine" | cut -d':' -f1) + + # Find the next timing line with the same process ID that comes after this process line + matchingTiming=$(awk -F: -v pid="$processId" -v startLine="$processLineNum" ' + $1 > startLine && $2 ~ "^"pid"," && /cpu time DFA/ { + print $0 + exit + } + ' "$timingLines") + + if [ -n "$matchingTiming" ]; then + cpuTime=$(getCpuTime "$matchingTiming") + printf "%2d. %-30s (PID: %s) → CPU time: %ss\n" "$counter" "$simName" "$processId" "$cpuTime" + ((counter++)) + fi + +done < "$processLines" + +echo +echo "Summary:" +echo "========" +echo "Total matched pairs: $((counter-1))" + +# Count simulations per process ID +echo "Process distribution:" +for pid in $(cut -d',' -f1 "$processLines" | cut -d':' -f2 | sort -u); do + count=$(grep -c "^[0-9]*:$pid," "$processLines") + echo " Process $pid: $count simulations" +done + +# Clean up temporary files +rm "$processLines" "$timingLines" + +echo +echo "Script completed successfully!" diff --git a/tinyHelper/logHelper_extractProcessTimesAndParams.sh b/tinyHelper/logHelper_extractProcessTimesAndParams.sh new file mode 100755 index 000000000..cbe187dc3 --- /dev/null +++ b/tinyHelper/logHelper_extractProcessTimesAndParams.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# Enhanced script to extract and match simulations with their parameters and timing +# Usage: ./extractProcessTimesAndParams + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + echo "Example: $0 runAna4ProbAna_20250917_14h34m46s.log" + exit 1 +fi + +logFile="$1" + +if [ ! -f "$logFile" ]; then + echo "Error: File '$logFile' not found!" + exit 1 +fi + +echo "Extracting simulation data with parameters from: $logFile" +echo "==================================================================" +echo + +# Create temporary files to store the extracted data +processLines=$(mktemp) +timingLines=$(mktemp) +thicknessLines=$(mktemp) +musamosatLines=$(mktemp) + +# Extract all relevant data with line numbers +grep -n "runs as process" "$logFile" > "$processLines" +grep -n "cpu time DFA" "$logFile" > "$timingLines" +grep -n "'relTh0':" "$logFile" > "$thicknessLines" +grep -n "Friction model parameter used: musamosat" "$logFile" > "$musamosatLines" + +# Functions to extract values +getProcessId() { + echo "$1" | cut -d',' -f1 | cut -d':' -f2 +} + +getSimName() { + echo "$1" | sed 's/.*INFO - \(.*\) runs as process.*/\1/' +} + +getCpuTime() { + echo "$1" | sed 's/.*cpu time DFA = \([0-9.]*\) s.*/\1/' +} + +getThickness() { + echo "$1" | sed "s/.*'relTh0': '\([0-9.]*\)'.*/\1/" +} + +getMusamosat() { + echo "$1" | sed 's/.*musamosat with value \([0-9.]*\).*/\1/' +} + +getLineNumber() { + echo "$1" | cut -d':' -f1 +} + +# Create arrays to store parameter values for each simulation +declare -A thicknessValues +declare -A musamosatValues +declare -A simCounter + +# Count simulations per process to track which parameter set belongs to which simulation +while IFS= read -r processLine; do + processId=$(getProcessId "$processLine") + if [[ -z "${simCounter[$processId]}" ]]; then + simCounter[$processId]=0 + else + ((simCounter[$processId]++)) + fi +done < "$processLines" + +# Reset counters for actual processing +declare -A currentSim + +# Process thickness values (relTh0) - these appear sequentially before simulations +thicknessIndex=0 +while IFS= read -r thicknessLine; do + thickness=$(getThickness "$thicknessLine") + thicknessValues["$thicknessIndex"]="$thickness" + ((thicknessIndex++)) +done < "$thicknessLines" + +# Reset simulation counters for musamosat processing +declare -A currentSimMu + +# Process musamosat values - these appear before each set of simulations +# We need to map them correctly to the simulation order +musamosatIndex=0 +while IFS= read -r musamosatLine; do + musamosat=$(getMusamosat "$musamosatLine") + musamosatValues["$musamosatIndex"]="$musamosat" + ((musamosatIndex++)) +done < "$musamosatLines" + +# Counter for matched pairs +counter=1 +musamosatSimIndex=0 +thicknessSimIndex=0 + +echo "Matched Simulations with Parameters and Timing:" +echo "==============================================" +printf "%-3s %-30s %-8s %-12s %-12s %-10s\n" "No." "Simulation Name" "PID" "μ_sat" "Thickness" "CPU Time" +printf "%-3s %-30s %-8s %-12s %-12s %-10s\n" "---" "------------------------------" "--------" "------------" "------------" "----------" + +# Read process lines and find matching data +while IFS= read -r processLine; do + processId=$(getProcessId "$processLine") + simName=$(getSimName "$processLine") + processLineNum=$(getLineNumber "$processLine") + + # Find matching timing line + matchingTiming=$(awk -F: -v pid="$processId" -v startLine="$processLineNum" ' + $1 > startLine && $2 ~ "^"pid"," && /cpu time DFA/ { + print $0 + exit + } + ' "$timingLines") + + if [ -n "$matchingTiming" ]; then + cpuTime=$(getCpuTime "$matchingTiming") + + # Get thickness for this simulation (sequential mapping) + thickness="${thicknessValues[$thicknessSimIndex]}" + + # Get musamosat for this simulation (they cycle through the parameter sets) + musamosat="${musamosatValues[$musamosatSimIndex]}" + + printf "%-3d %-30s %-8s %12.3f %12.3f %-10ss\n" "$counter" "$simName" "$processId" "$musamosat" "$thickness" "$cpuTime" + + ((counter++)) + ((musamosatSimIndex++)) + ((thicknessSimIndex++)) + + # Reset musamosat index if we've used all parameter sets + if [ "$musamosatSimIndex" -ge "$musamosatIndex" ]; then + musamosatSimIndex=0 + fi + fi + +done < "$processLines" + +echo +echo "Summary:" +echo "========" +echo "Total matched simulations: $((counter-1))" +echo "Parameter sets used:" +echo " Thickness values: $thicknessIndex" +echo " μ_sat values: $musamosatIndex" + +# Process distribution +echo "Process distribution:" +for pid in $(cut -d',' -f1 "$processLines" | cut -d':' -f2 | sort -u); do + count=$(grep -c "^[0-9]*:$pid," "$processLines") + echo " Process $pid: $count simulations" +done + +# Clean up temporary files +rm "$processLines" "$timingLines" "$thicknessLines" "$musamosatLines" + +echo +echo "Script completed successfully!" From 0dd10a372c181fbe8df5f44e7115573a6567a025 Mon Sep 17 00:00:00 2001 From: Felix Oesterle Date: Thu, 18 Sep 2025 10:38:46 +0200 Subject: [PATCH 2/2] refactor(com1DFA): remove unused `modName` from parameter list --- avaframe/com1DFA/com1DFA.py | 1 - 1 file changed, 1 deletion(-) diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index 9da4586a4..f7f6aa3fd 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -2975,7 +2975,6 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting [ relNameSim, simHash, - modName, defID, frictIndi or volIndi, row._asdict()["simTypeList"],