data-machinelearning-the-boring-way/ok.yml at main · abhishek-ch/data-machinelearning-the-boring-way · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
  name: user
  namespace: default
spec:
  sparkConf:
    "spark.ui.port": "4045"
    "spark.eventLog.enabled": "false"
    "spark.hadoop.fs.s3a.access.key": "abhishek"
    "spark.hadoop.fs.s3a.secret.key": "choudhary123"
    "spark.hadoop.fs.s3a.endpoint": "http://minio:9000"
    "spark.hadoop.fs.s3n.impl": "org.apache.hadoop.fs.s3n.S3AFileSystem"
    "spark.hadoop.fs.s3n.fast.upload": "true"
    "spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version": "2"
    "spark.hadoop.fs.s3a.connection.ssl.enabled": "false"
    "spark.hadoop.fs.s3a.path.style.access": "true"
  type: Python
  pythonVersion: "3"
  mode: cluster
  image: "pysparkexample:aws2"
  imagePullPolicy: Never
  mainApplicationFile: local:///app/pyspark_jobs/accountbypartition.py
  arguments:
    - s3a://test-files/sample/output/users
    - s3a://test-files/sample/output/users_account"
  sparkVersion: "3.1.1"
  restartPolicy:
    type: OnFailure
    onFailureRetries: 0
    onFailureRetryInterval: 1
    onSubmissionFailureRetries: 0
    onSubmissionFailureRetryInterval: 1
  driver:
    coreRequest: "500m"
    coreLimit: "1000m"
    memory: "512m"
    labels:
      version: 3.1.1
    serviceAccount: spark
  executor:
    coreRequest: "250m"
    instances: 2
    memory: "512m"
    labels:
      version: 3.1.1