kubectl -n rook-ceph exec -it deploy/rook-ceph-tools-operator-image -- bash

export AWS_HOST=rook-ceph-rgw-my-store.rook-ceph.svc      # 如果使用 K8s 内地址，可以使用该地址，如果使用是外部 rook-ceph，可以使用 IP 或域名
export PORT=800
export BUCKET_NAME=ceph-bkt-8c8cfba0-e4f4-4302-a2e5-d693d243e948
export AWS_ACCESS_KEY_ID=Q2HLJBVD739N9XOR3SNZ
export AWS_SECRET_ACCESS_KEY=pevdQm9d82AR1vAuIrVmpUNSth0G6nkQ5KzmmJdG

mkdir ~/.aws
cat > ~/.aws/credentials << EOF
[default]
aws_access_key_id = ${AWS_ACCESS_KEY_ID}
aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}
EOF

echo "Hello Rook" > /tmp/rookObj
s5cmd --endpoint-url http://$AWS_HOST:$PORT cp /tmp/rookObj s3://$BUCKET_NAME

s5cmd --endpoint-url http://$AWS_HOST:$PORT cp s3://$BUCKET_NAME/rookObj /tmp/rookObj-download
cat /tmp/rookObj-download

run_mode = shared_data          # 存算分离模式
cloud_native_meta_port = 6090    # 云原生元数据服务监听端口，默认端口 6090
cloud_native_storage_type = S3  # 存储类型 S3
aws_s3_path = ceph-bkt-8c8cfba0-e4f4-4302-a2e5-d693d243e948    # 此处选择 bucket 名称
aws_s3_endpoint = rook-ceph-rgw-my-store.rook-ceph.svc.cluster.local:800  # endpoint 即 http://IP+port，rook-ceph-rgw-my-store.rook-ceph.svc.cluster.local 写法为 K8s 内部域名格式，应当前 rook-ceph 和 starrocks 在同一个集群内部，采用 K8s 内部域名，我的对象存储配置端口为 800
aws_s3_access_key = Q2HLJBVD739N9XOR3SNZ        # access_key，每个 bucket 唯一
aws_s3_secret_key = pevdQm9d82AR1vAuIrVmpUNSth0G6nkQ5KzmmJdG    # secret_key，每个 bucket 唯一
enable_load_volume_from_conf = true  # true 表示 StarRocks 将使用 FE 配置文件中存储相关属性创建内置存储卷 builtin_storage_volume，并将其设置为默认存储卷。但如果没有指定存储相关的属性，StarRocks 将无法启动。

...
100     # config for share data mode
101     run_mode = shared_data
102     cloud_native_meta_port = 6090
103     cloud_native_storage_type = S3
104     aws_s3_path = ceph-bkt-8c8cfba0-e4f4-4302-a2e5-d693d243e948
105     #aws_s3_region = <region>   # 公有云 aws 需要改参数，ceph 对接不需要该参数
106     aws_s3_endpoint = rook-ceph-rgw-my-store.rook-ceph.svc.cluster.local:800
107     aws_s3_access_key = Q2HLJBVD739N9XOR3SNZ
108     aws_s3_secret_key = pevdQm9d82AR1vAuIrVmpUNSth0G6nkQ5KzmmJdG
109     # create the default storage volume manually after the cluster is created
110     enable_load_volume_from_conf = true

curl -O https://raw.githubusercontent.com/StarRocks/starrocks-kubernetes-operator/main/deploy/starrocks.com_starrocksclusters.yaml
kubectl apply -f starrocks.com_starrocksclusters.yaml

curl -O https://raw.githubusercontent.com/StarRocks/starrocks-kubernetes-operator/main/deploy/operator.yaml

kubectl apply -f operator.yaml

kubectl -n starrocks get pod

grep -n namespace operator.yaml

curl -O https://raw.githubusercontent.com/StarRocks/starrocks-kubernetes-operator/main/examples/starrocks/starrocks-fe-and-be.yaml
kubectl apply -f starrocks-fe-and-be.yaml

cat > shared-data-fe-cn-feproxy-v1.0.yaml <<-'EOF'
# This manifest deploys a StarRocks cluster running in shared data mode.
# see https://docs.starrocks.io/en-us/latest/deployment/deploy_shared_data for more information about share data mode.
#
# You will have to download and edit this YAML file to specify the details for your shared storage. See the
# examples in the docs, and add your customizations to the ConfigMap `starrockscluster-sample-fe-cm` at the
# bottom of this file.
# https://docs.starrocks.io/en-us/latest/deployment/deploy_shared_data#configure-fe-nodes-for-shared-data-starrocks
apiVersion: starrocks.com/v1
kind: StarRocksCluster
metadata:
  name: a-starrocks-in-share-data-mode   # change the name if needed.
  namespace: starrocks
spec:
  starRocksFeSpec:     # 部署 FE 部分
    image: starrocks/fe-ubuntu:latest     # 镜像版本
    replicas: 1   # 当前测试环境启动 1 个 pod，可以后续作为扩容验证，也可启动 3 个
    limits:
      cpu: 4         # 调小所需的资源，生产环境可调大
      memory: 6Gi
    requests:
      cpu: 1
      memory: 2Gi
    configMapInfo:
      configMapName: starrockscluster-sample-fe-cm   # 引用 configmap 作为 fe.conf 内容，可以作为后续参数优化使用，完整内容在后面
      resolveKey: fe.conf
    storageVolumes:
    - name: fe-storage-meta
      storageClassName: local-storage     # 使用 LocalPath 作为 fe 持久化数据存储
      # fe container stop running if the disk free space which the fe meta directory residents, is less than 5Gi.
      storageSize: 10Gi
      mountPath: /opt/starrocks/fe/meta   # fe meta 持久化数据目录
    - name: fe-storage-log
      storageClassName: local-storage
      storageSize: 5Gi
      mountPath: /opt/starrocks/fe/log   # fe log 持久化数据目录

  starRocksCnSpec:     # 部署 CN 部分
#    cnEnvVars:   # 调试时使用参数
#      - name: LOG_CONSOLE
#        value: "1"
    image: starrocks/cn-ubuntu:latest     # 镜像版本
    replicas: 1   # 当前测试环境启动 1 个 pod，可以后续作为扩容验证，也可启动 3 个
    limits:
      cpu: 4           # 调小所需的资源，生产环境可调大
      memory: 6Gi
    requests:
      cpu: 1
      memory: 2Gi
    configMapInfo:
      configMapName: starrockscluster-sample-cn-cm   # 引用 configmap 作为 cn.conf 内容，可以作为后续参数优化使用，完整内容在后面
      resolveKey: cn.conf
    storageVolumes:
    - name: cn-storage-cache
      storageClassName: local-storage          # 使用 LocalPath 作为 cn 持久化数据存储
      # fe container stop running if the disk free space which the fe meta directory residents, is less than 5Gi.
      storageSize: 200Gi
      mountPath: /opt/starrocks/cn/storage   # cn 缓存等持久化数据目录

  starRocksFeProxySpec:     # 部署 FE Proxy 部分
    image: nginx:latest     # 镜像版本
    replicas: 1   # 当前测试环境启动 1 个 pod，可以后续作为扩容验证，也可启动 3 个
    limits:
      cpu: 1           # 调小所需的资源，生产环境可调大
      memory: 2Gi
    requests:
      cpu: 200m
      memory: 500Mi
    service:
      type: NodePort   # export fe proxy service
      ports:
        - containerPort: 8080
          name: http-port
          nodePort: 30180   # The range of valid ports is 30000-32767
          port: 8080
    resolver: "kube-dns.kube-system.svc.cluster.local"  # this is the default dns server.

---
# fe config
apiVersion: v1
kind: ConfigMap
metadata:
  name: starrockscluster-sample-fe-cm
  namespace: starrocks
  labels:
    cluster: starrockscluster-sample
data:
  fe.conf: |
    LOG_DIR = ${STARROCKS_HOME}/log
    DATE = "$(date +%Y%m%d-%H%M%S)"
    # 降低 JVM 内存大小为 4G: Xmx4096m, 共 3 处
    JAVA_OPTS="-Dlog4j2.formatMsgNoLookups=true -Xmx4096m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:${LOG_DIR}/fe.gc.log.$DATE"
    JAVA_OPTS_FOR_JDK_9="-Dlog4j2.formatMsgNoLookups=true -Xmx4096m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:${LOG_DIR}/fe.gc.log.$DATE:time"
    JAVA_OPTS_FOR_JDK_11="-Dlog4j2.formatMsgNoLookups=true -Xmx4096m -XX:+UseG1GC -Xlog:gc*:${LOG_DIR}/fe.gc.log.$DATE:time"
    http_port = 8030
    rpc_port = 9020
    query_port = 9030
    edit_log_port = 9010
    mysql_service_nio_enabled = true
    sys_log_level = INFO
    # config for share data mode
    # 将之前 S3 的信息填充到下方
    run_mode = shared_data
    cloud_native_meta_port = 6090
    cloud_native_storage_type = S3
    aws_s3_path = ceph-bkt-8c8cfba0-e4f4-4302-a2e5-d693d243e948
    #aws_s3_region = <region>
    aws_s3_endpoint = rook-ceph-rgw-my-store.rook-ceph.svc.cluster.local:800
    aws_s3_access_key = Q2HLJBVD739N9XOR3SNZ
    aws_s3_secret_key = pevdQm9d82AR1vAuIrVmpUNSth0G6nkQ5KzmmJdG
    # create the default storage volume manually after the cluster is created
    enable_load_volume_from_conf = true

---
# cn config
apiVersion: v1
kind: ConfigMap
metadata:
  name: starrockscluster-sample-cn-cm
  namespace: starrocks
  labels:
    cluster: starrockscluster-sample
data:
  cn.conf: |
    starlet_port = 9070
    storage_root_path = /opt/starrocks/cn/storage
    #block_cache_mem_size = 2147483648
    #block_cache_disk_size = 21474836480
EOF

cat > localStorageClassV1.0.yaml << -'EOF'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: local-storage      # StorageClass 名称，可自定义
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: starrocks-fe-meta-pv     # fe-meta pv 名称
  namespace: starrocks
spec:
  capacity:
    storage: 10Gi             # fe meta 大小
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage     # 使用上述定义的 SC 名称
  local:
    path: /opt/starrocks/fe/meta      # fe meta 本机路径
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - k8s-sr1    # 后续 fe-meta 固定在该节点，如果有多个节点，则更改每个节点主机名，需要 label 进行标记

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: starrocks-fe-log-pv    # fe-log pv 名称
  namespace: starrocks
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage
  local:
    path: /opt/starrocks/fe/log         # fe log 本机路径
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - k8s-sr1

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: starrocks-cn-cache-pv     # cn-cache pv 名称
  namespace: starrocks
spec:
  capacity:
    storage: 200Gi         # cn cache 个人建议 200G - 2T
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage
  local:
    path: /opt/starrocks/cn/storage     # cn cache 本机路径
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - k8s-sr1
EOF

kubectl apply -f shared-data-fe-cn-feproxy-v1.0.yaml

kubectl apply -f localStorageClassV1.0.yaml

kubectl -n starrocks get pod

kubectl -n starrocks describe pod a-starrocks-in-share-data-mode-fe-proxy-857455f4df-pbdpv

kubectl -n starrocks edit deployments.apps/a-starrocks-in-share-data-mode-fe-proxy

starRocksFeProxySpec:     # 部署 FE Proxy 部分
  image: nginx:latest     # 镜像版本

kubectl -n starrocks get svc

kubectl -n starrocks exec -it a-starrocks-in-share-data-mode-fe-0 -- bash

mysql -h 11.111.196.31 -P 9030 -uroot

mysql -h a-starrocks-in-share-data-mode-fe-service -P 9030 -uroot

SHOW PROC '/frontends'\G
SHOW PROC '/compute_nodes'\G

starRocksFeSpec:
  image: starrocks/fe-ubuntu:3.0-latest
  replicas: 3
  requests:
    cpu: 4
    memory: 16Gi
  service:
    type: LoadBalancer # 指定为 LoadBalancer

kubectl -n starrocks get svc

mysql -h a7509284bf3784983a596c6eec7fc212-618xxxxxx.us-west-2.elb.amazonaws.com -P9030 -uroot

cat > fe-nodeport-svc.yaml <<-'EOF'
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: fe                     # svc label, --show-labels
    app.starrocks.ownerreference/name: a-starrocks-in-share-data-mode
  name: starrocks-fe-nodeport
  namespace: starrocks
spec:
  ports:
  - name: http
    port: 8030
    protocol: TCP
    targetPort: 8030
    nodePort: 32030    # http 8030 对外访问端口 32030
  - name: query
    port: 9030
    protocol: TCP
    targetPort: 9030
    nodePort: 32630    # query 8030 对外访问端口 32630
  selector:
    app.kubernetes.io/component: fe             # pod label, --show-tables
    app.starrocks.ownerreference/name: a-starrocks-in-share-data-mode-fe
  type: NodePort
EOF

kubectl apply -f fe-nodeport-svc.yaml

kubectl -n starrocks get svc

mysql -h192.168.80.20 -P32630 -uroot