minikube etcd 조각 모음 defrag

root@k8s-master1 manifests]# find / -type f -name etcdctl 2>/dev/null
/run/containerd/io.containerd.runtime.v1.linux/k8s.io/4fc80ceb99dfc0dca39e726d95104f5e424c53e618fd71d201b9b8b9c75a6d5d/rootfs/usr/local/bin/etcdctl
/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/9/fs/usr/local/bin/etcdctl

etcdctl을 /usr/bin/ 밑으로 복사


#!/bin/bash

alias etcdctl3='ETCDCTL_API=3 etcdctl --cacert=/var/lib/minikube/certs/etcd/ca.crt --cert=/var/lib/minikube/certs/etcd/server.crt --key=/var/lib/minikube/certs/etcd/server.key'

root@minikube home]# etcdctl3 -w table endpoint status --cluster
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.45.100:2379 | d5b1b2d93f592f08 |   3.5.0 |  1.6 MB |      true |      false |         4 |      46687 |              46687 |        |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

## 알람제거 ##
etcdctl3 --endpoints="https://${endpoint}:2379" alarm disarm
etcdctl3 --endpoints="https://${endpoint}:2379" alarm list

각 서버에서 실행하기 위해서는 
etcdctl3 alarm disarm
etcdctl3 alarm list

########### 압축하기 ####

etcdctl3 endpoint status --write-out="json" | egrep -o '"revision":[0-9]*'| egrep -o '[0-9].*'
"revision":41713

etcdctl3 compact 41713
#######################

## 조각 모음하기 ####
== 클러스터 전체 동시 ===
[root@minikube home]# etcdctl3 defrag --cluster    #클러스터 전체를 동시에
Finished defragmenting etcd member[https://192.168.45.100:2379]


[root@minikube home]# etcdctl3 -w table endpoint status --cluster
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.45.100:2379 | d5b1b2d93f592f08 |   3.5.0 |  856 kB |      true |      false |         4 |      48305 |              48305 |        |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

=== etcd서버 하나씩 조각 모음 : 서비스 영향확인 필요 ==

etcdctl3 -w table endpoint status --cluster 

리더를 확인한다. 리더를 제외한 나머지 부분에서 진행하며, 리더는 맨 마지막에 진행 

etcdctl3 defrag --endpoints="https://${endpoint}:2379"
etcdctl3 --endpoints="https://192.168.45.100:2379" --write-out=table endpoint status
해당서버에서 etcd pod가 정상적으로 올라오는지 확인한다
- 반복 --



# set a very small 16MB quota
$ etcd.yaml에  --quota-backend-bytes=$((16*1024*1024))  #계산된 값을 넣어준다


# fill keyspace
$ while [ 1 ]; do dd if=/dev/urandom bs=1024 count=1024  | etcdctl3 put key  || break; done
...
Error:  rpc error: code = 8 desc = etcdserver: mvcc: database space exceeded
# confirm quota space is exceeded

[root@minikube manifests]# etcdctl3 -w table endpoint status --cluster                  +-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------------------------------+
|          ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX |             ERRORS             |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------------------------------+
| https://192.168.45.100:2379 | d5b1b2d93f592f08 |   3.5.0 |   16 MB |      true |      false |         5 |      50310 |              50310 |  memberID:15398285247096893192 |
|                             |                  |         |         |           |            |           |            |                    |                 alarm:NOSPACE  |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------------------------------+
[root@minikube manifests]# etcdctl3 alarm list
memberID:15398285247096893192 alarm:NOSPACE

[root@minikube manifests]# etcdctl3 alarm disarm
memberID:15398285247096893192 alarm:NOSPACE

[root@minikube manifests]# etcdctl3 alarm list

## etcd.yaml에서 추가 ###

============ 압축하기 ============
 keep one hour of history
 --auto-compaction-retention=1


======용량 늘리기 ==================
예제: 8G  DEFAULT: 2G이며 MAX: 8G이다

--quota-backend-bytes=8589934592 


###################  job으로 조각모음 하기  :  etcd 2G기준   ########

[root@minikube manifests]#  etcdctl3 -w table endpoint status --cluster
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.45.100:2379 | d5b1b2d93f592f08 |   3.5.0 |  408 MB |      true |      false |         2 |       1144 |               1144 |        |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
kubectl label node minikube etcd="true"

[root@minikube home]#  etcdctl3 --endpoints="https://192.168.45.100:2379" --write-out=table endpoint status;
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.45.100:2379 | d5b1b2d93f592f08 |   3.5.0 |  176 MB |      true |      false |         2 |       2819 |               2819 |        |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

 kubectl apply -f minikube-etcd-defrag-job.yaml

minikube-etcd-defrag-job.yaml

0.00MB

[root@minikube home]# etcdctl --endpoints="https://192.168.45.100:2379" --write-out=table endpoint status;
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.45.100:2379 | d5b1b2d93f592f08 |   3.5.0 |  2.0 MB |      true |      false |         2 |       3456 |               3456 |        |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

## 스냅샷 파일 정보보기 ##

$ etcdctl snapshot save backup.db
$ etcdctl --write-out=table snapshot status backup.db
+----------+----------+------------+------------+
|   HASH   | REVISION | TOTAL KEYS | TOTAL SIZE |
+----------+----------+------------+------------+
| fe01cf57 |       10 |          7 | 2.1 MB     |
+----------+----------+------------+------------+

## db 파일에 직업 조각 모음 하기
  - --data-dir=/var/lib/minikube/etcd


[root@minikube snap]# etcdctl3 --endpoints="https://192.168.45.100:2379" --write-out=table endpoint status;
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.45.100:2379 | d5b1b2d93f592f08 |   3.5.0 |  5.2 MB |      true |      false |         2 |      23546 |              23546 |        |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@minikube snap]# etcdctl3 defrag  /var/lib/minikube/etcd
Finished defragmenting etcd member[127.0.0.1:2379]
[root@minikube snap]# etcdctl3 --endpoints="https://192.168.45.100:2379" --write-out=table endpoint status;
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.45.100:2379 | d5b1b2d93f592f08 |   3.5.0 |  1.8 MB |      true |      false |         2 |      23564 |              23564 |        |
+-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@minikube snap]# ls -al
total 3792
drwx------ 2 root root     108 Nov 21 15:26 .
drwx------ 4 root root      29 Nov 21 08:29 ..
-rw-r--r-- 1 root root    7148 Nov 21 11:15 0000000000000002-0000000000002711.snap
-rw-r--r-- 1 root root    7148 Nov 21 14:20 0000000000000002-0000000000004e22.snap
-rw------- 1 root root 1843200 Nov 21 15:27 db
[root@minikube snap]#


##이건 뭘까 ? ##
 ./etcdctl3 check datascale --load="s" --auto-compact=true --auto-defrag=true