kubectl -n rook-ceph exec -it deploy/rook-ceph-tools-operator-image -- bash

export AWS_HOST=rook-ceph-rgw-my-store.rook-ceph.svc      # 如果使用K8s内地址，可以使用该地址，如果使用是外部rook-ceph，可以使用IP或域名
export PORT=800
export BUCKET_NAME=ceph-bkt-8c8cfba0-e4f4-4302-a2e5-d693d243e948
export AWS_ACCESS_KEY_ID=Q2HLJBVD739N9XOR3SNZ
export AWS_SECRET_ACCESS_KEY=pevdQm9d82AR1vAuIrVmpUNSth0G6nkQ5KzmmJdG

mkdir ~/.aws
cat > ~/.aws/credentials << EOF
[default]
aws_access_key_id = ${AWS_ACCESS_KEY_ID}
aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}
EOF

echo "Hello Rook" > /tmp/rookObj
s5cmd --endpoint-url http://$AWS_HOST:$PORT cp /tmp/rookObj s3://$BUCKET_NAME

s5cmd --endpoint-url http://$AWS_HOST:$PORT cp s3://$BUCKET_NAME/rookObj /tmp/rookObj-download
cat /tmp/rookObj-download

run_mode = shared_data          # 存算分离模式
cloud_native_meta_port = 6090    # 云原生元数据服务监听端口，默认端口6090
cloud_native_storage_type = S3  # 存储类型S3
aws_s3_path = ceph-bkt-8c8cfba0-e4f4-4302-a2e5-d693d243e948    # 此处选择bucket名称
aws_s3_endpoint = rook-ceph-rgw-my-store.rook-ceph.svc.cluster.local:800  # endpoint即http://IP+port
aws_s3_access_key = Q2HLJBVD739N9XOR3SNZ  # access_key，每个bucket唯一
aws_s3_secret_key = pevdQm9d82AR1vAuIrVmpUNSth0G6nkQ5KzmmJdG  # secret_key，每个bucket唯一
enable_load_volume_from_conf = true  # true表示StarRocks将使用FE配置文件中存储相关属性创建内置存储卷

curl -O https://raw.githubusercontent.com/StarRocks/starrocks-kubernetes-operator/main/deploy/starrocks.com_starrocksclusters.yaml
kubectl apply -f starrocks.com_starrocksclusters.yaml

curl -O https://raw.githubusercontent.com/StarRocks/starrocks-kubernetes-operator/main/deploy/operator.yaml

kubectl apply -f operator.yaml

kubectl -n starrocks get pod

cat > shared-data-fe-cn-feproxy-v1.0.yaml <<-'EOF'
# This manifest deploys a StarRocks cluster running in shared data mode.
# see https://docs.starrocks.io/en-us/latest/deployment/deploy_shared_data for more information about share data mode.
#
# You will have to download and edit this YAML file to specify the details for your shared storage. See the
# examples in the docs, and add your customizations to the ConfigMap `starrockscluster-sample-fe-cm` at the
# bottom of this file.
# https://docs.starrocks.io/en-us/latest/deployment/deploy_shared_data#configure-fe-nodes-for-shared-data-starrocks
apiVersion: starrocks.com/v1
kind: StarRocksCluster
metadata:
  name: a-starrocks-in-share-data-mode   # change the name if needed.
  namespace: starrocks
spec:
  starRocksFeSpec:     #部署Fe部分
    image: starrocks/fe-ubuntu:latest     #镜像版本
    replicas: 1   #当前测试环境启动1个pod,可以后续作为扩容验证,也可启动3个
    limits:
      cpu: 4         #调小所需的资源，生产环境可调大
      memory: 6Gi
    requests:
      cpu: 1
      memory: 2Gi
    configMapInfo:
      configMapName: starrockscluster-sample-fe-cm   #引用configmap作为fe.conf内容,可以作为后续参数优化使用,完整内容在后面
      resolveKey: fe.conf
    storageVolumes:
    - name: fe-storage-meta
      storageClassName: local-storage     #使用LocalPath作为fe持久化数据存储
      # fe container stop running if the disk free space which the fe meta directory residents, is less than 5Gi.
      storageSize: 10Gi
      mountPath: /opt/starrocks/fe/meta   #fe meta持久化数据目录
    - name: fe-storage-log
      storageClassName: local-storage
      storageSize: 5Gi
      mountPath: /opt/starrocks/fe/log   #fe log持久化数据目录

  starRocksCnSpec:     #部署Fe部分
#    cnEnvVars:   #调试时使用参数
#      - name: LOG_CONSOLE
#        value: "1"
    image: starrocks/cn-ubuntu:latest     #镜像版本
    replicas: 1   #当前测试环境启动1个pod,可以后续作为扩容验证,也可启动3个
    limits:
      cpu: 4           #调小所需的资源，生产环境可调大
      memory: 6Gi
    requests:
      cpu: 1
      memory: 2Gi
    configMapInfo:
      configMapName: starrockscluster-sample-cn-cm   #引用configmap作为cn.conf内容,可以作为后续参数优化使用,完整内容在后面
      resolveKey: cn.conf
    storageVolumes:
    - name: cn-storage-cache
      storageClassName: local-storage          #使用LocalPath作为cn持久化数据存储
      # fe container stop running if the disk free space which the fe meta directory residents, is less than 5Gi.
      storageSize: 200Gi
      mountPath: /opt/starrocks/cn/storage   #cn 缓存等持久化数据目录

  starRocksFeProxySpec:     #部署FeProxy部分
    image: nginx:latest     #镜像版本
    replicas: 1   #当前测试环境启动1个pod,可以后续作为扩容验证,也可启动3个
    limits:
      cpu: 1           #调小所需的资源，生产环境可调大
      memory: 2Gi
    requests:
      cpu: 200m
      memory: 500Mi
    service:
      type: NodePort   # export fe proxy service
      ports:
        - containerPort: 8080
          name: http-port
          nodePort: 30180   # The range of valid ports is 30000-32767
          port: 8080
    resolver: "kube-dns.kube-system.svc.cluster.local"  # this is the default dns server.

---
# fe config
apiVersion: v1
kind: ConfigMap
metadata:
  name: starrockscluster-sample-fe-cm
  namespace: starrocks
  labels:
    cluster: starrockscluster-sample
data:
  fe.conf: |
    LOG_DIR = ${STARROCKS_HOME}/log
    DATE = "$(date +%Y%m%d-%H%M%S)"
    # 降低JVM内存大小为4G:Xmx4096m,共3处
    JAVA_OPTS="-Dlog4j2.formatMsgNoLookups=true -Xmx4096m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:${LOG_DIR}/fe.gc.log.$DATE"
    JAVA_OPTS_FOR_JDK_9="-Dlog4j2.formatMsgNoLookups=true -Xmx4096m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:${LOG_DIR}/fe.gc.log.$DATE:time"
    JAVA_OPTS_FOR_JDK_11="-Dlog4j2.formatMsgNoLookups=true -Xmx4096m -XX:+UseG1GC -Xlog:gc*:${LOG_DIR}/fe.gc.log.$DATE:time"
    http_port = 8030
    rpc_port = 9020
    query_port = 9030
    edit_log_port = 9010
    mysql_service_nio_enabled = true
    sys_log_level = INFO
    # config for share data mode
    # 将之前S3的信息填充到下方
    run_mode = shared_data
    cloud_native_meta_port = 6090
    cloud_native_storage_type = S3
    aws_s3_path = ceph-bkt-8c8cfba0-e4f4-4302-a2e5-d693d243e948
    #aws_s3_region = <region>
    aws_s3_endpoint = rook-ceph-rgw-my-store.rook-ceph.svc.cluster.local:800
    aws_s3_access_key = Q2HLJBVD739N9XOR3SNZ
    aws_s3_secret_key = pevdQm9d82AR1vAuIrVmpUNSth0G6nkQ5KzmmJdG
    # create the default storage volume manually after the cluster is created
    enable_load_volume_from_conf = true

---
# cn config
apiVersion: v1
kind: ConfigMap
metadata:
  name: starrockscluster-sample-cn-cm
  namespace: starrocks
  labels:
    cluster: starrockscluster-sample
data:
  cn.conf: |
    starlet_port = 9070
    storage_root_path = /opt/starrocks/cn/storage
    #block_cache_mem_size = 2147483648
    #block_cache_disk_size = 21474836480
EOF

cat > localStorageClassV1.0.yaml << -'EOF'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: local-storage      #StorageClass名称，可自定义
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: starrocks-fe-meta-pv     #fe-meta pv名称
  namespace: starrocks
spec:
  capacity:
    storage: 10Gi             #fe meta大小
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage     #使用上述定义的SC名称
  local:
    path: /opt/starrocks/fe/meta      #fe meta本机路径
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - k8s-sr1    #后续fe-meta固定在该节点，如果有多个节点，则更改每个节点主机名，需要label进行标记

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: starrocks-fe-log-pv    #fe-log pv名称
  namespace: starrocks
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage
  local:
    path: /opt/starrocks/fe/log         #fe log本机路径
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - k8s-sr1

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: starrocks-cn-cache-pv     #cn-cache pv名称
  namespace: starrocks
spec:
  capacity:
    storage: 200Gi         #cn cache个人建议200G - 2T
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage
  local:
    path: /opt/starrocks/cn/storage     #cn cache本机路径
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - k8s-sr1
EOF

kubectl apply -f shared-data-fe-cn-feproxy-v1.0.yaml

kubectl apply -f localStorageClassV1.0.yaml

kubectl -n starrocks get pod

kubectl -n starrocks describe pod a-starrocks-in-share-data-mode-fe-proxy-857455f4df-pbdpv

kubectl -n starrocks get svc

kubectl -n starrocks exec -it a-starrocks-in-share-data-mode-fe-0 -- bash

mysql -h 11.111.196.31 -P 9030 -uroot

mysql -h a-starrocks-in-share-data-mode-fe-service -P 9030 -uroot

SHOW PROC '/frontends'\G
SHOW PROC '/compute_nodes'\G

kubectl -n starrocks edit svc a-starrocks-in-share-data-mode-fe-service

starRocksFeSpec:
  image: starrocks/fe-ubuntu:3.0-latest
  replicas: 3
  requests:
    cpu: 4
    memory: 16Gi
  service:
    type: LoadBalancer # 指定为LoadBalancer

kubectl -n starrocks get svc

mysql -h a7509284bf3784983a596c6eec7fc212-618xxxxxx.us-west-2.elb.amazonaws.com -P9030 -uroot

cat > fe-nodeport-svc.yaml <<-'EOF'
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: fe                     # svc lable, --show-labels
    app.starrocks.ownerreference/name: a-starrocks-in-share-data-mode
  name: starrocks-fe-nodeport
  namespace: starrocks
spec:
  ports:
  - name: http
    port: 8030
    protocol: TCP
    targetPort: 8030
    nodePort: 32030    #http 8030对外访问端口32030
  - name: query
    port: 9030
    protocol: TCP
    targetPort: 9030
    nodePort: 32630    #query 8030对外访问端口32630
  selector:
    app.kubernetes.io/component: fe             # pod label, --show-tables
    app.starrocks.ownerreference/name: a-starrocks-in-share-data-mode-fe
  type: NodePort
EOF

kubectl apply -f fe-nodeport-svc.yaml

kubectl -n starrocks get svc

mysql -h192.168.80.20 -P32630 -uroot