Skip to content

Commit 581bef9

Browse files
justonedev1justonedev1
committed
[OCTRL-1091] control-workflows manifests for readout and dd to be used by k8s bridge
1 parent bc2aa06 commit 581bef9

4 files changed

Lines changed: 166 additions & 1 deletion

File tree

‎control-operator/ecs-manifests/control-workflows/readout-docker.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ defaults:
77
_module_cmdline: >-
88
source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load Readout Control-OCCPlugin &&
99
o2-readout-exe
10-
_plain_cmdline: "sudo -E docker run --name readout --replace --privileged --user flp -v /etc/group:/etc/group:ro -v /etc/passwd:/etc/passwd:ro -v /tmp:/tmp -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) --network=host --ipc=host -e O2_DETECTOR -e O2_PARTITION -e OCC_CONTROL_PORT -e O2_SYSTEM -e O2_ROLE gitlab-registry.cern.ch/aliceo2group/dockerfiles/alma9-flp-node:2 /opt/o2/bin/o2-readout-exe"
10+
_plain_cmdline: "sudo -E docker run --name readout --replace --privileged --user flp -v /etc/group:/etc/group:ro -v /etc/passwd:/etc/passwd:ro -v /tmp:/tmp -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) --network=host --ipc=host -e O2_DETECTOR -e O2_PARTITION -e OCC_CONTROL_PORT -e O2_SYSTEM -e O2_ROLE gitlab-registry.cern.ch/aliceo2group/dockerfiles/alma9-flp-node/readout:1 /opt/o2/bin/o2-readout-exe"
1111
control:
1212
mode: direct
1313
wants:

‎control-operator/ecs-manifests/control-workflows/readout-kube-direct.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: readout # read by workflow
2+
defaults:
3+
readout_cfg_uri: "consul-ini://{{ consul_endpoint }}/o2/components/readout/ANY/any/readout-standalone-{{ task_hostname }}"
4+
user: flp
5+
log_task_stdout: none
6+
log_task_stderr: none
7+
_module_cmdline: >-
8+
source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load Readout Control-OCCPlugin &&
9+
o2-readout-exe
10+
_plain_cmdline: "{{ o2_install_path }}/bin/o2-readout-exe"
11+
control:
12+
mode: kubernetes_direct
13+
wants:
14+
cpu: 0.15
15+
memory: 128
16+
bind:
17+
- name: readout
18+
type: push
19+
rateLogging: "{{ fmq_rate_logging }}"
20+
addressing: ipc
21+
transport: shmem
22+
properties: {}
23+
command:
24+
stdout: "{{ log_task_stdout }}"
25+
stderr: "{{ log_task_stderr }}"
26+
shell: true
27+
env:
28+
- O2_DETECTOR={{ detector }}
29+
- O2_PARTITION={{ environment_id }}
30+
user: "{{ user }}"
31+
arguments:
32+
- "{{ readout_cfg_uri }}"
33+
value: "{{ len(modulepath)>0 ? _module_cmdline : _plain_cmdline }}"

‎control-operator/ecs-manifests/control-workflows/stfbuilder-senderoutput-kube-direct.yaml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: stfbuilder-senderoutput
2+
control:
3+
mode: kubernetes_fairmq
4+
defaults:
5+
detector: TST
6+
rdh_version: "6"
7+
user: flp
8+
fmq_severity: info
9+
dd_detector_subspec: feeid
10+
dd_discovery_endpoint: "no-op://" # location of consul docker instance
11+
log_task_stdout: none
12+
log_task_stderr: none
13+
stfb_dd_mode: "physics"
14+
stfb_fee_mask: "0xffff"
15+
stfb_enable_datasink: "false"
16+
stfb_datasink_dir: "/tmp"
17+
stfb_datasink_filesize: "2048"
18+
monitoring_dd_interval: "5"
19+
_module_cmdline: >-
20+
source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load DataDistribution Control-OCCPlugin &&
21+
StfBuilder
22+
_plain_cmdline: "{{ o2_install_path }}/bin/StfBuilder"
23+
wants:
24+
cpu: 0.15
25+
memory: 128
26+
bind:
27+
- name: buildertosender
28+
type: push
29+
rateLogging: "{{ fmq_rate_logging }}"
30+
transport: shmem
31+
addressing: ipc
32+
sndBufSize: "4"
33+
command:
34+
env:
35+
- O2_INFOLOGGER_MODE={{infologger_mode}}
36+
- O2_DETECTOR={{ detector }}
37+
- DATADIST_FEE_MASK={{ stfb_fee_mask }}
38+
- O2_PARTITION={{ environment_id }}
39+
stdout: "{{ log_task_stdout }}"
40+
stderr: "{{ log_task_stderr }}"
41+
shell: true
42+
user: "{{ user }}"
43+
value: "{{ len(modulepath)>0 ? _module_cmdline : _plain_cmdline }}"
44+
arguments:
45+
- "--session=default"
46+
- "--transport=shmem"
47+
- "--shm-segment-id=2"
48+
- "--shm-segment-size=33554432"
49+
- "--monitoring-backend={{ monitoring_dd_url }}"
50+
- "--monitoring-process-interval={{ monitoring_dd_interval }}"
51+
- "--discovery-partition={{ environment_id }}"
52+
- "--discovery-endpoint={{ dd_discovery_endpoint }}"
53+
- "--detector-rdh={{ rdh_version }}"
54+
- "--detector-subspec={{ dd_detector_subspec }}"
55+
- "--severity={{ fmq_severity }}"
56+
- "--severity-infologger={{ infologger_severity }}"
57+
- "--output-channel-name=buildertosender"
58+
- "--shm-monitor=false"
59+
- "--detector={{ detector }}"
60+
- "--run-type={{ stfb_dd_mode }}"
61+
- "{{ stfb_enable_datasink == 'true' ? '--data-sink-enable' : ' ' }}"
62+
- "{{ stfb_enable_datasink == 'true' ? '--data-sink-dir=' + stfb_datasink_dir : ' ' }}"
63+
- "{{ stfb_enable_datasink == 'true' ? '--data-sink-max-stfs-per-file=0' : ' ' }}"
64+
- "{{ stfb_enable_datasink == 'true' ? '--data-sink-max-file-size=' + stfb_datasink_filesize : ' ' }}"

‎control-operator/ecs-manifests/control-workflows/stfsender-kube-direct.yaml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: stfsender
2+
control:
3+
mode: kubernetes_fairmq
4+
wants:
5+
cpu: 0.15
6+
memory: 128
7+
defaults:
8+
max_buffered_stfs: "-1"
9+
dd_discovery_net_if: "ib0" #the loopback intf
10+
dd_discovery_endpoint: "no-op://" # location of consul docker instance
11+
user: flp
12+
fmq_severity: info
13+
log_task_stdout: none
14+
log_task_stderr: none
15+
stfs_dd_region_size: 4096
16+
stfs_shm_segment_size: 33554432
17+
stfs_enable_datasink: "false"
18+
stfs_datasink_dir: "/tmp"
19+
stfs_datasink_filesize: "2048"
20+
monitoring_dd_interval: "5"
21+
_module_cmdline: >-
22+
source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load DataDistribution Control-OCCPlugin &&
23+
numactl --cpunodebind=0 --preferred=0 -- StfSender
24+
_plain_cmdline: "numactl"
25+
command:
26+
env:
27+
- O2_INFOLOGGER_MODE={{infologger_mode}}
28+
- http_proxy=""
29+
- https_proxy=""
30+
- no_proxy=-ib,.internal
31+
- O2_DETECTOR={{ detector }}
32+
- UCX_NET_DEVICES=mlx5_0:1 # This limits StfSender to IB interface (used as of DD v1.3.0)
33+
- UCX_TLS=sm,self,dc,rc # Force dc/rc connection (used as of DD v1.4.0)
34+
- UCX_IB_SL=1 # Force IB SL1 with Adaptive Routing (AR)
35+
- UCX_DC_MAX_NUM_EPS=512 # Allow 512 peers for DC transport
36+
- UCX_RC_MAX_NUM_EPS=512 # Allow 512 peers for RC transport
37+
- O2_PARTITION={{ environment_id }}
38+
stdout: "{{ log_task_stdout }}"
39+
stderr: "{{ log_task_stderr }}"
40+
shell: true
41+
user: "{{ user }}"
42+
value: "{{ len(modulepath)>0 ? _module_cmdline : _plain_cmdline }}"
43+
arguments:
44+
- "--cpunodebind=0"
45+
- "--preferred=0"
46+
- "--"
47+
- "{{ o2_install_path }}/bin/StfSender"
48+
- "--session=default"
49+
- "--shm-segment-id=2"
50+
- "--shm-segment-size={{ stfs_shm_segment_size }}"
51+
- "--dd-region-size={{ stfs_dd_region_size }}"
52+
- "--dd-region-id=3536"
53+
- "--transport=shmem"
54+
- "--input-channel-name={{ stfs_input_channel_name }}"
55+
- "--severity={{ fmq_severity }}"
56+
- "--severity-infologger={{ infologger_severity }}"
57+
- "--monitoring-backend={{ monitoring_dd_url }}"
58+
- "--monitoring-process-interval={{ monitoring_dd_interval }}"
59+
- "--discovery-partition={{ environment_id }}"
60+
- "--discovery-id={{ dd_discovery_stfs_id }}"
61+
- "--discovery-endpoint={{ dd_discovery_endpoint }}"
62+
- "--discovery-net-if={{ dd_discovery_net_if }}"
63+
- '{{ ddsched_enabled == "true" ? "" : "--stand-alone" }}'
64+
- "--shm-monitor=false"
65+
- "{{ stfs_enable_datasink == 'true' ? '--data-sink-enable' : ' ' }}"
66+
- "{{ stfs_enable_datasink == 'true' ? '--data-sink-dir=' + stfs_datasink_dir : ' ' }}"
67+
- "{{ stfs_enable_datasink == 'true' ? '--data-sink-max-stfs-per-file=0' : ' ' }}"
68+
- "{{ stfs_enable_datasink == 'true' ? '--data-sink-max-file-size=' + stfs_datasink_filesize : ' ' }}"

0 commit comments

Comments
 (0)