From 168304d9cd302cf7e1e3df0a983331703d190ece Mon Sep 17 00:00:00 2001 From: Muryanto Date: Wed, 28 Oct 2020 14:15:05 -0700 Subject: [PATCH 1/3] add helm install and ansible install for wget API service --- deploy/ansible/roles/index/defaults/main.yml | 18 +++++++ deploy/ansible/roles/index/tasks/main.yml | 8 +++ .../roles/index/tasks/wget_api_install.yml | 34 +++++++++++++ .../roles/index/tasks/wget_api_uninstall.yml | 12 +++++ .../index/templates/esgf_wgetapi_config.j2 | 25 ++++++++++ deploy/ansible/roles/proxy/defaults/main.yml | 5 ++ .../roles/proxy/templates/proxy.conf.j2 | 8 +++ .../chart/templates/ingress/ingress.yaml | 6 +++ .../chart/templates/wgetApi/Deployment.yaml | 49 +++++++++++++++++++ .../chart/templates/wgetApi/configmap.yaml | 31 ++++++++++++ .../chart/templates/wgetApi/secret.yaml | 10 ++++ .../chart/templates/wgetApi/service.yaml | 16 ++++++ deploy/kubernetes/chart/values.yaml | 40 +++++++++++++++ 13 files changed, 262 insertions(+) create mode 100644 deploy/ansible/roles/index/tasks/wget_api_install.yml create mode 100644 deploy/ansible/roles/index/tasks/wget_api_uninstall.yml create mode 100644 deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 create mode 100644 deploy/kubernetes/chart/templates/wgetApi/Deployment.yaml create mode 100644 deploy/kubernetes/chart/templates/wgetApi/configmap.yaml create mode 100644 deploy/kubernetes/chart/templates/wgetApi/secret.yaml create mode 100644 deploy/kubernetes/chart/templates/wgetApi/service.yaml diff --git a/deploy/ansible/roles/index/defaults/main.yml b/deploy/ansible/roles/index/defaults/main.yml index aa10c192..6ef0bd44 100644 --- a/deploy/ansible/roles/index/defaults/main.yml +++ b/deploy/ansible/roles/index/defaults/main.yml @@ -68,3 +68,21 @@ search_image_prefix: "{{ image_prefix }}" search_image_tag: "{{ image_tag }}" search_image_pull: "{{ image_pull }}" search_image_repository: search + +### +# wgetApi configuration +### +wget_api_enabled: true +wget_api_image_prefix: "{{ image_prefix }}" +wget_api_tag: "{{ image_tag }}" +wget_api_image_pull: "{{ image_pull }}" +wget_api_image_repository: wget_api +wget_api: + debug: False + allowed_hosts: "{{ wget_api.allowed_hosts }}" + solr_url: "{{ wget_api.solr_url }}" + shards_xml: "{{ wget_api.shards_xml }}" + script_file_default_limit: "{{ wget_api.script_file_default_limit }}" + script_file_max_limit: "{{ wget_api.script_file_max_limit }}" + data_upload_max_number_fields: "{{ wget_api.data_upload_max_number_fields }}" + diff --git a/deploy/ansible/roles/index/tasks/main.yml b/deploy/ansible/roles/index/tasks/main.yml index 7cd8eade..642d6ca2 100644 --- a/deploy/ansible/roles/index/tasks/main.yml +++ b/deploy/ansible/roles/index/tasks/main.yml @@ -22,3 +22,11 @@ - name: Uninstall search application include: search_uninstall.yml when: "'index' not in group_names or not search_enabled" + +- name: Install wget_api application + include: wget_api_install.yml + when: "'index' in group_names and wget_api_enabled" + +- name: Uninstall wget_api application + include: wget_api_uninstall.yml + when: "'index' not in group_names or not wget_api_enabled" diff --git a/deploy/ansible/roles/index/tasks/wget_api_install.yml b/deploy/ansible/roles/index/tasks/wget_api_install.yml new file mode 100644 index 00000000..679100f8 --- /dev/null +++ b/deploy/ansible/roles/index/tasks/wget_api_install.yml @@ -0,0 +1,34 @@ +--- + +- name: Create Docker network + docker_network: + name: esgf + +- name: Make wget api config directory + file: + path: /tmp/esgf_wget + state: directory + +- name: Write wget api local settings + template: + src: "esgf_wgetapi_config.j2" + dest: "/tmp/esgf_wget/{{ wget_api.config }}" + +- name: Start wget_api container + docker_container: + name: wget_api + env: + ESGF_WGET_CONFIG: "{{ wget_api.config_path }}/{{ wget_api.config }}" + ESGF_WGET_SECRET_KEY: "{{ wget_api.secret_key }}" + image: "{{ wget_api_image_prefix }}/{{ wget_api_image_repository }}:{{ wget_api_image_tag }}" + pull: "{{ wget_api_image_pull }}" + detach: yes + restart_policy: unless-stopped + exposed_ports: + - "8000" + networks: + - name: esgf + networks_cli_compatible: yes + volumes: ["/tmp/esgf_wget:{{ wget_api.config_path }}:ro"] + state: started + restart: yes diff --git a/deploy/ansible/roles/index/tasks/wget_api_uninstall.yml b/deploy/ansible/roles/index/tasks/wget_api_uninstall.yml new file mode 100644 index 00000000..54224539 --- /dev/null +++ b/deploy/ansible/roles/index/tasks/wget_api_uninstall.yml @@ -0,0 +1,12 @@ +--- + +- name: Stop wget_api container + docker_container: + name: wget_api + state: absent + +- name: Remove wget api config directory + file: + path: /tmp/esgf_wget + state: absent + diff --git a/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 b/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 new file mode 100644 index 00000000..64d44a3b --- /dev/null +++ b/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 @@ -0,0 +1,25 @@ +[django] + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ALLOWED_HOSTS = {{ wget_api.allowed_hosts }} + +# Expand the number of fields allowed for wget API +DATA_UPLOAD_MAX_NUMBER_FIELDS = 1024 + +[wget] +# Address of ESGF Solr +ESGF_SOLR_URL = {{ wget_api.solr_url }} + +# Path to XML file containing Solr shards +ESGF_SOLR_SHARDS_XML = {{ wget_api.shards_xml }} + +# Default limit on the number of files allowed in a wget script +WGET_SCRIPT_FILE_DEFAULT_LIMIT = {{ wget_api.script_file_default_limit }} + +# Maximum number of files allowed in a wget script +WGET_SCRIPT_FILE_MAX_LIMIT = {{ wget_api.script_file_max_limit }} + + + diff --git a/deploy/ansible/roles/proxy/defaults/main.yml b/deploy/ansible/roles/proxy/defaults/main.yml index a7d389b7..4221b17e 100644 --- a/deploy/ansible/roles/proxy/defaults/main.yml +++ b/deploy/ansible/roles/proxy/defaults/main.yml @@ -43,3 +43,8 @@ solr_enabled: true # Indicates if search is enabled search_enabled: true + +# Indicates if wget_api is enabled +wget_api_enabled: true + + diff --git a/deploy/ansible/roles/proxy/templates/proxy.conf.j2 b/deploy/ansible/roles/proxy/templates/proxy.conf.j2 index 4c06cd2e..37ee8a8b 100644 --- a/deploy/ansible/roles/proxy/templates/proxy.conf.j2 +++ b/deploy/ansible/roles/proxy/templates/proxy.conf.j2 @@ -43,5 +43,13 @@ server { proxy_pass http://search:8080; } {% endif %} + + {% if wget_api_enabled %} + location /wget { + include /etc/nginx/includes/proxy_params.conf; + proxy_pass http://wget_api:8000; + } + {% endif %} {% endif %} } + diff --git a/deploy/kubernetes/chart/templates/ingress/ingress.yaml b/deploy/kubernetes/chart/templates/ingress/ingress.yaml index 8c8bf84a..f24affee 100644 --- a/deploy/kubernetes/chart/templates/ingress/ingress.yaml +++ b/deploy/kubernetes/chart/templates/ingress/ingress.yaml @@ -54,4 +54,10 @@ spec: serviceName: {{ include "esgf.component.fullname" (list . "search") }} servicePort: 8080 {{- end }} + {{- if .Values.index.wgetApi.enabled }} + - path: /wget + backend: + serviceName: {{ include "esgf.component.fullname" (list . "wget-api") }} + servicePort: 8000 + {{- end }} {{- end }} diff --git a/deploy/kubernetes/chart/templates/wgetApi/Deployment.yaml b/deploy/kubernetes/chart/templates/wgetApi/Deployment.yaml new file mode 100644 index 00000000..816bb9be --- /dev/null +++ b/deploy/kubernetes/chart/templates/wgetApi/Deployment.yaml @@ -0,0 +1,49 @@ +{{- $wgetApi := .Values.index.wgetApi -}} +{{- if (and .Values.index.enabled $wgetApi.enabled) -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "esgf.component.fullname" (list . "wgetApi") }} + labels: {{ include "esgf.component.labels" (list . "wgetApi" $wgetApi.labels) | nindent 4 }} +spec: + replicas: {{ $wgetApi.replicaCount }} + selector: + matchLabels: {{ include "esgf.component.selectorLabels" (list . "wgetApi") | nindent 6 }} + template: + metadata: + labels: {{ include "esgf.component.selectorLabels" (list . "wgetApi") | nindent 8 }} + spec: + {{- with $wgetApi.podSecurityContext }} + securityContext: {{ toYaml . | nindent 8 }} + {{- end }} + containers: + - name: wget-api + {{ include "esgf.deployment.image" (list . $wgetApi.image) }} + env: + - name: ESGF_WGET_CONFIG + value: {{ $wgetApi.config_path }}/{{ $wgetApi.config }} + - name: ESGF_WGET_SECRET_KEY + valueFrom: + secretKeyRef: + name: wget-api-secret + key: secret + ports: + - name: http + containerPort: 8000 + readinessProbe: &probe + httpGet: + path: /wget + port: 8000 + initialDelaySeconds: 20 + periodSeconds: 20 + livenessProbe: + <<: *probe + initialDelaySeconds: 30 + volumeMounts: + - name: config + mountPath: {{ $wgetApi.config_path }} + volumes: + - name: config + configMap: + name: {{ include "esgf.component.fullname" (list . "wgetApi") }} +{{- end -}} diff --git a/deploy/kubernetes/chart/templates/wgetApi/configmap.yaml b/deploy/kubernetes/chart/templates/wgetApi/configmap.yaml new file mode 100644 index 00000000..ea7524f5 --- /dev/null +++ b/deploy/kubernetes/chart/templates/wgetApi/configmap.yaml @@ -0,0 +1,31 @@ +{{- $wgetApi := .Values.index.wgetApi -}} +{{- if (and .Values.index.enabled $wgetApi.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "esgf.component.fullname" (list . "wgetApi") }} + labels: {{ include "esgf.component.labels" (list . "wgetApi" $wgetApi.labels) | nindent 4 }} +data: + wget_api_config: | + [django] + DEBUG = {{ $wgetApi.settings.debug }} + + ALLOWED_HOSTS = {{ $wgetApi.settings.allowedHosts }} + + # Expand the number of fields allowed for wget API + DATA_UPLOAD_MAX_NUMBER_FIELDS = {{ $wgetApi.settings.dataUploadMaxNumberFields }} + + [wget] + # Address of ESGF Solr + ESGF_SOLR_URL = {{ $wgetApi.settings.esgfSolrUrl }} + + # Path to XML file containing Solr shards + ESGF_SOLR_SHARDS_XML = {{ $wgetApi.settings.esgfSolrShardsXml }} + + # Default limit on the number of files allowed in a wget script + WGET_SCRIPT_FILE_DEFAULT_LIMIT = {{ $wgetApi.settings.wgetScriptFileDefaultLimit }} + + # Maximum number of files allowed in a wget script + WGET_SCRIPT_FILE_MAX_LIMIT = {{ $wgetApi.settings.wgetScriptFileMaxLimit }} + +{{- end -}} diff --git a/deploy/kubernetes/chart/templates/wgetApi/secret.yaml b/deploy/kubernetes/chart/templates/wgetApi/secret.yaml new file mode 100644 index 00000000..ea5a997a --- /dev/null +++ b/deploy/kubernetes/chart/templates/wgetApi/secret.yaml @@ -0,0 +1,10 @@ +{{- $wgetApi := .Values.index.wgetApi -}} +{{- if (and .Values.index.enabled $wgetApi.enabled) -}} +apiVersion: v1 +kind: Secret +metadata: + name: wget-api-secret +type: Opaque +data: + secret: {{ randAlphaNum 50 | b64enc }} +{{- end -}} diff --git a/deploy/kubernetes/chart/templates/wgetApi/service.yaml b/deploy/kubernetes/chart/templates/wgetApi/service.yaml new file mode 100644 index 00000000..6526bfe5 --- /dev/null +++ b/deploy/kubernetes/chart/templates/wgetApi/service.yaml @@ -0,0 +1,16 @@ +{{- $wgetApi := .Values.index.wgetApi -}} +{{- if (and .Values.index.enabled $wgetApi.enabled) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "esgf.component.fullname" (list . "wgetApi") }} + labels: {{ include "esgf.component.labels" (list . "wgetApi" $wgetApi.labels) | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + targetPort: 8000 + protocol: TCP + selector: {{ include "esgf.component.selectorLabels" (list . "wgetApi") | nindent 4 }} +{{- end -}} diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index d57eba26..ee42498e 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -299,3 +299,43 @@ index: # The tolerations for the search pods # See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ tolerations: + + # Configuration for wgetApi + wgetApi: + # Indicates if wgetApi component should be deployed + enabled: true + # Image overrides for the wgetApi image + image: + # prefix: + repository: esgf-wget + # tag: + # The pod security context for all wgetApi pods + # This should normally not need to change unless Solr pods are configured to use + # specific paths on the host for core storage + podSecurityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + # The container security context for all Solr containers + securityContext: + # Run with a read-only root filesystem by default + readOnlyRootFilesystem: true + # The absolute path where wget api config file will be stored. + config_path: /esg/esgf_wget + # The wget api config file name to be created under config_path + config: wget_api_config + settings: + debug: False + allowedHosts: "*" + # ESGF solr url. For ex: https://esgf-node.llnl.gov/solr + esgfSolrUrl: + # Path to XML file containing Solr shards. For ex: /esg/config/esgf_shards_static.xml + esgfSolrShardsXml: + # Default limit on the number of files allowed in a wget script + wgetScriptFileDefaultLimit: 1000 + # Maximum number of files allowed in a wget script + wgetScriptFileMaxLimit: 100000 + # Expand the number of fields allowed for wget API + dataUploadMaxNumberFields: 10240 + # The number of replicas for the wgetApi pod. + replicaCount: 1 From 89df348aed532222e91cd000156b4f0f644af434 Mon Sep 17 00:00:00 2001 From: Muryanto Date: Fri, 8 Jan 2021 14:37:06 -0800 Subject: [PATCH 2/3] update to ansible deploy for wgetapi --- .../roles/index/tasks/wget_api_install.yml | 10 +++++++++ .../esgf_wgetapi_allowed_projects.json.j2 | 3 +++ .../index/templates/esgf_wgetapi_config.j2 | 8 +++++-- .../esgf_wgetapi_solr_shards_static.xml.j2 | 6 ++++++ .../chart/templates/wgetApi/configmap.yaml | 21 ++++++++++++++++++- 5 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2 create mode 100644 deploy/ansible/roles/index/templates/esgf_wgetapi_solr_shards_static.xml.j2 diff --git a/deploy/ansible/roles/index/tasks/wget_api_install.yml b/deploy/ansible/roles/index/tasks/wget_api_install.yml index 679100f8..e6ac2eb1 100644 --- a/deploy/ansible/roles/index/tasks/wget_api_install.yml +++ b/deploy/ansible/roles/index/tasks/wget_api_install.yml @@ -13,6 +13,16 @@ template: src: "esgf_wgetapi_config.j2" dest: "/tmp/esgf_wget/{{ wget_api.config }}" + +- name: Write wget api allowed projects + template: + src: "esgf_wgetapi_allowed_projects.json.j2" + dest: "/tmp/esgf_wget/{{ wget_api.allowed_projects_json }}" + +- name: Write XML file containing Solr shards + template: + src: "esgf_wgetapi_solr_shards_static.xml.j2" + dest: "/tmp/esgf_wget/esgf_wgetapi_solr_shards_static.xml" - name: Start wget_api container docker_container: diff --git a/deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2 b/deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2 new file mode 100644 index 00000000..6daaf7f8 --- /dev/null +++ b/deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2 @@ -0,0 +1,3 @@ +{ + "allowed_projects": {{ wget_api.allowed_projects | list }} +} \ No newline at end of file diff --git a/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 b/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 index 64d44a3b..a0281adc 100644 --- a/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 +++ b/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 @@ -13,7 +13,10 @@ DATA_UPLOAD_MAX_NUMBER_FIELDS = 1024 ESGF_SOLR_URL = {{ wget_api.solr_url }} # Path to XML file containing Solr shards -ESGF_SOLR_SHARDS_XML = {{ wget_api.shards_xml }} +ESGF_SOLR_SHARDS_XML = {{ wget_api.config_path }}/esgf_wgetapi_solr_shards_static.xml + +# Path to JSON file containing allowed projects to access for datasets +ESGF_ALLOWED_PROJECTS_JSON = {{ wget_api.config_path }}/{{ wget_api.allowed_projects_json }} # Default limit on the number of files allowed in a wget script WGET_SCRIPT_FILE_DEFAULT_LIMIT = {{ wget_api.script_file_default_limit }} @@ -21,5 +24,6 @@ WGET_SCRIPT_FILE_DEFAULT_LIMIT = {{ wget_api.script_file_default_limit }} # Maximum number of files allowed in a wget script WGET_SCRIPT_FILE_MAX_LIMIT = {{ wget_api.script_file_max_limit }} - +# Maximum length for facet values used in the wget directory structure +WGET_MAX_DIR_LENGTH = {{ wget_api.max_dir_length }} diff --git a/deploy/ansible/roles/index/templates/esgf_wgetapi_solr_shards_static.xml.j2 b/deploy/ansible/roles/index/templates/esgf_wgetapi_solr_shards_static.xml.j2 new file mode 100644 index 00000000..fa393b25 --- /dev/null +++ b/deploy/ansible/roles/index/templates/esgf_wgetapi_solr_shards_static.xml.j2 @@ -0,0 +1,6 @@ + + + {% for item in wget_api.solr_shards %} + {{ item }} + {% endfor %} + diff --git a/deploy/kubernetes/chart/templates/wgetApi/configmap.yaml b/deploy/kubernetes/chart/templates/wgetApi/configmap.yaml index ea7524f5..64f8fd87 100644 --- a/deploy/kubernetes/chart/templates/wgetApi/configmap.yaml +++ b/deploy/kubernetes/chart/templates/wgetApi/configmap.yaml @@ -22,10 +22,29 @@ data: # Path to XML file containing Solr shards ESGF_SOLR_SHARDS_XML = {{ $wgetApi.settings.esgfSolrShardsXml }} + # Path to JSON file containing allowed projects to access for datasets + ESGF_ALLOWED_PROJECTS_JSON = {{ $wgetApi.config_path}}/{{ $wgetApi.allowed_projects_json }} + # Default limit on the number of files allowed in a wget script WGET_SCRIPT_FILE_DEFAULT_LIMIT = {{ $wgetApi.settings.wgetScriptFileDefaultLimit }} # Maximum number of files allowed in a wget script WGET_SCRIPT_FILE_MAX_LIMIT = {{ $wgetApi.settings.wgetScriptFileMaxLimit }} - + esgf_allowed_projects.json: | + { + "allowed_projects": [{{ $wgetApi.settings.allowed_projects | join "," }}] + } + esgf_solr_shards.xml: | + + + + {{ range $wgetApi.settings.shards }} + {{ . }} + {{ end }} + {{- end -}} From 2789f353c4ca98ba4158551b8fa9f89d4e71101a Mon Sep 17 00:00:00 2001 From: Muryanto Date: Mon, 11 Jan 2021 17:34:41 -0800 Subject: [PATCH 3/3] fixed allowed_projects setting in the deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2, and make esgf-wget DEBUG to be settable --- .../index/templates/esgf_wgetapi_allowed_projects.json.j2 | 4 ++-- deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2 b/deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2 index 6daaf7f8..847779fe 100644 --- a/deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2 +++ b/deploy/ansible/roles/index/templates/esgf_wgetapi_allowed_projects.json.j2 @@ -1,3 +1,3 @@ { - "allowed_projects": {{ wget_api.allowed_projects | list }} -} \ No newline at end of file + "allowed_projects": ["{{ wget_api.allowed_projects | join('", "') }}"] +} diff --git a/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 b/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 index a0281adc..13fc36f5 100644 --- a/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 +++ b/deploy/ansible/roles/index/templates/esgf_wgetapi_config.j2 @@ -1,7 +1,7 @@ [django] # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = {{ wget_api.debug }} ALLOWED_HOSTS = {{ wget_api.allowed_hosts }}