wget https://dlcdn.apache.org/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz
tar -xvzf spark-3.2.2-bin-hadoop3.2.tgz

## Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ARG java_image_tag=11-jre-slim
FROM openjdk:${java_image_tag}
ARG spark_uid=185

# Before building the docker image, first build and make a Spark distribution following
# the instructions in http://spark.apache.org/docs/latest/building-spark.html.
# If this docker file is being used in the context of building your images from a Spark
# distribution, the docker build command should be invoked from the top level directory
# of the Spark distribution. E.g.:
# docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile .

RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y sudo
RUN set -ex && 
    sed -i 's/http://deb.(.*)/https://deb.1/g' /etc/apt/sources.list && 
    apt-get update && 
    ln -s /lib /lib64 && 
    apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps && 
    mkdir -p /opt/spark && 
    mkdir -p /opt/spark/examples && mkdir -p /opt/spark/conf && 
    mkdir -p /usr/local/jdk1.8.0_162 && 
    mkdir -p /opt/spark/work-dir && 
    touch /opt/spark/RELEASE && 
    rm /bin/sh && 
    ln -sv /bin/bash /bin/sh && 
    echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && 
    chgrp root /etc/passwd && chmod ug+rw /etc/passwd && 
    rm -rf /var/cache/apt/*

COPY jars /opt/spark/jars
COPY bin /opt/spark/bin
COPY sbin /opt/spark/sbin
COPY kubernetes/dockerfiles/spark/entrypoint.sh /opt/
COPY kubernetes/dockerfiles/spark/decom.sh /opt/
COPY examples /opt/spark/examples
COPY kubernetes/tests /opt/spark/tests
COPY data /opt/spark/data
COPY jdk1.8.0_162 /usr/local/jdk1.8.0_162/
COPY hosts /opt/spark/
COPY conf /opt/spark/conf
ENV SPARK_HOME /opt/spark
WORKDIR /opt/spark/work-dir
RUN chmod g+w /opt/spark/work-dir
RUN chmod a+x /opt/decom.sh
ENTRYPOINT [ "/opt/entrypoint.sh" ]

# Specify the User that the actual main process will run as
# USER ${spark_uid}
USER root

sudo ./bin/docker-image-tool.sh -r lfspace/thpub -t thspark3.2.2 build

WARNING: Ignoring http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/APKINDEX.tar.gz: temporary error (try again later)
WARNING: Ignoring http://dl-cdn.alpinelinux.org/alpine/v3.9/community/x86_64/APKINDEX.tar.gz: temporary error (try again later)
ERROR: unsatisfiable constraints:  bash (missing):
    required by: world[bash]

docker tag lfspace/thpub/spark:thspark3.2.2 lfspace/thpub:spark3.2.2
docker push lfspace/thpub:spark3.2.2

bin/spark-submit 
    --master k8s://https://<k8s-apiserver-host>:<k8s-apiserver-port> 
    --deploy-mode cluster 
    --name spark-pi 
    --class org.apache.spark.examples.SparkPi 
    --conf spark.executor.instances=5 
    --conf spark.kubernetes.container.image=<spark-image> 
    local:///path/to/examples.jar

apiVersion: v1
kind: ServiceAccount
metadata:
  name: spark
  namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  namespace: default
  name: spark-role
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["*"]
- apiGroups: [""]
  resources: ["services"]
  verbs: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: spark-role-binding
  namespace: default
subjects:
- kind: ServiceAccount
  name: spark
  namespace: default
roleRef:
  kind: Role
  name: spark-role
  apiGroup: rbac.authorization.k8s.io

sudo kubectl apply -f role.yaml

sudo kubectl get role
sudo kubectl get role spark-role -o yaml
sudo kubectl get rolebinding
sudo kubectl get rolebinding spark-role-binding -o yaml

kubectl create serviceaccount spark
kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:spark --namespace=default

apiVersion: apps/v1
kind: Deployment
metadata:
  name: spark-client
spec:
  replicas: 1
  selector:
    matchLabels:
      app: spark
      component: client
  template:
    metadata:
      labels:
        app: spark
        component: client
    spec:
      containers:
      - name: sparkclient
        image: lfspace/thpub:spark3.2.2
        workingDir: /opt/spark
        command: ["/bin/bash", "-c", "while true;do echo hello;sleep 6000;done"]
      serviceAccountName: spark

sudo kubectl create -f client-deployment.yaml

sudo kubectl exec -t -i spark-client-6479b76776-l5bzw /bin/bash

TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
curl --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 
     -H "Authorization: Bearer $TOKEN" 
     -s https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}/api/v1/namespaces/default/pods

- apiGroups: [""]
  resources: ["*"]
  verbs: ["*"]

# 第二个 wordcount.py 是作为参数用
bin/spark-submit 
    --master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} 
    --deploy-mode cluster 
    --name spark-test 
    --conf spark.executor.instances=3 
    --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark 
    --conf spark.kubernetes.container.image=lfspace/thpub:spark3.2.2 
    /opt/spark/examples/src/main/python/wordcount.py 
    /opt/spark/examples/src/main/python/wordcount.py

javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target

2020/10/15 11:09:49.147 WARN WatchConnectionManager : Exec Failure: HTTP 403, Status: 403 - pods "spark-pi-1602731387162-driver" is forbidden: User "system:anonymous" cannot watch resource "pods" in API group "" in the namespace "default"

--conf spark.kubernetes.authenticate.submission.clientKeyFile=/root/admin-key.pem
--conf spark.kubernetes.authenticate.submission.clientCertFile=/root/admin.pem

kubectl create clusterrolebinding test:anonymous --clusterrole=cluster-admin --user=system:anonymous

--conf spark.kubernetes.authenticate.driver.serviceAccountName=spark

/Users/zhouwenyang/Desktop/tmp/spark/bin/spark-submit 
--master k8s://https://knode1:6443 --deploy-mode cluster 
--name spark-pi --class org.apache.spark.examples.SparkPi 
--conf spark.executor.instances=2 
--conf spark.kubernetes.container.image=vm2173:5000/spark:2.4.7 
--conf spark.kubernetes.authenticate.driver.serviceAccountName=spark 
--conf spark.kubernetes.submission.waitAppCompletion=false 
local:///opt/spark/examples/jars/spark-examples_2.11-2.4.7.jar