In my Kubernetes cluster:
root@k8s-eu-1-control-plane-node-1:~# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-eu-1-control-plane-node-1 Ready control-plane 125d v1.28.4
k8s-eu-1-control-plane-node-2 Ready control-plane 125d v1.28.4
k8s-eu-1-control-plane-node-3 Ready control-plane 125d v1.28.4
k8s-eu-1-worker-1 Ready <none> 125d v1.28.4
k8s-eu-1-worker-2 Ready <none> 125d v1.28.4
k8s-eu-1-worker-3 Ready <none> 125d v1.28.4
I’m trying to set up and make it work a dgraph cluster.
I’ve created the following configuration files:
dgraph-local-storage-class.yaml :
root@k8s-eu-1-control-plane-node-1:~# nano DgraphConfig/dgraph-local-storage-class.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: local-storage-dgraph
namespace: dgraph
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
dgraph-pv-alpha-control-plane-node-1.yaml :
root@k8s-eu-1-control-plane-node-1:~# cat ./DgraphConfig/dgraph-pv-alpha-control-plane-node-1.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: dgraph-pv-alpha-control-plane-node-1
namespace: dgraph
spec:
storageClassName: local-storage-dgraph
local:
path: /root/dgraph/alpha
capacity:
storage: 350Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- k8s-eu-1-control-plane-node-1
dgraph-pv-alpha-control-plane-node-2.yaml :
root@k8s-eu-1-control-plane-node-1:~# cat ./DgraphConfig/dgraph-pv-alpha-control-plane-node-2.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: dgraph-pv-alpha-control-plane-node-2
namespace: dgraph
spec:
storageClassName: local-storage-dgraph
local:
path: /root/dgraph/alpha
capacity:
storage: 350Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- k8s-eu-1-control-plane-node-2
dgraph-pv-alpha-control-plane-node-3.yaml :
root@k8s-eu-1-control-plane-node-1:~# cat ./DgraphConfig/dgraph-pv-alpha-control-plane-node-3.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: dgraph-pv-alpha-control-plane-node-3
namespace: dgraph
spec:
storageClassName: local-storage-dgraph
local:
path: /root/dgraph/alpha
capacity:
storage: 350Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- k8s-eu-1-control-plane-node-3
dgraph-pv-zero-control-worker-node-1.yaml :
root@k8s-eu-1-control-plane-node-1:~# cat ./DgraphConfig/dgraph-pv-zero-control-worker-node-1.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: dgraph-pv-zero-worker-node-1
namespace: dgraph
spec:
storageClassName: local-storage-dgraph
local:
path: /root/dgraph/zero
capacity:
storage: 200Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- k8s-eu-1-worker-1
dgraph-pv-zero-control-worker-node-2.yaml :
root@k8s-eu-1-control-plane-node-1:~# cat ./DgraphConfig/dgraph-pv-zero-control-worker-node-2.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: dgraph-pv-zero-worker-node-2
namespace: dgraph
spec:
storageClassName: local-storage-dgraph
local:
path: /root/dgraph/zero
capacity:
storage: 200Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- k8s-eu-1-worker-2
dgraph-pv-zero-control-worker-node-3.yaml :
root@k8s-eu-1-control-plane-node-1:~# cat ./DgraphConfig/dgraph-pv-zero-control-worker-node-3.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: dgraph-pv-zero-worker-node-3
namespace: dgraph
spec:
storageClassName: local-storage-dgraph
local:
path: /root/dgraph/zero
capacity:
storage: 200Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- k8s-eu-1-worker-3
dgraph-ha.yaml :
root@k8s-eu-1-control-plane-node-1:~# cat DgraphConfig/dgraph-ha/dgraph-ha.yaml
# This highly available config creates 3 Dgraph Zeros, 3 Dgraph
# Alphas with 3 replicas, and 1 Ratel UI client. The Dgraph cluster
# will still be available to service requests even when one Zero
# and/or one Alpha are down.
#
# There are 3 services can can be used to expose outside the cluster as needed:
# dgraph-zero-public - To load data using Live & Bulk Loaders
# dgraph-alpha-public - To connect clients and for HTTP APIs
# dgraph-ratel-public - For Dgraph UI
apiVersion: v1
kind: Service
metadata:
name: dgraph-zero-public
namespace: dgraph
labels:
app: dgraph-zero
monitor: zero-dgraph-io
spec:
type: ClusterIP
ports:
- port: 5080
targetPort: 5080
name: grpc-zero
- port: 6080
targetPort: 6080
name: http-zero
selector:
app: dgraph-zero
---
apiVersion: v1
kind: Service
metadata:
name: dgraph-alpha-public
labels:
app: dgraph-alpha
monitor: alpha-dgraph-io
spec:
type: ClusterIP
ports:
- port: 8080
targetPort: 8080
name: http-alpha
- port: 9080
targetPort: 9080
name: grpc-alpha
selector:
app: dgraph-alpha
---
apiVersion: v1
kind: Service
metadata:
name: dgraph-ratel-public
labels:
app: dgraph-ratel
spec:
type: ClusterIP
ports:
- port: 8000
targetPort: 8000
name: http-ratel
selector:
app: dgraph-ratel
---
# This is a headless service which is necessary for discovery for a dgraph-zero StatefulSet.
# https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#creating-a-statefulset
apiVersion: v1
kind: Service
metadata:
name: dgraph-zero
labels:
app: dgraph-zero
spec:
ports:
- port: 5080
targetPort: 5080
name: grpc-zero
clusterIP: None
# We want all pods in the StatefulSet to have their addresses published for
# the sake of the other Dgraph Zero pods even before they're ready, since they
# have to be able to talk to each other in order to become ready.
publishNotReadyAddresses: true
selector:
app: dgraph-zero
---
# This is a headless service which is necessary for discovery for a dgraph-alpha StatefulSet.
# https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#creating-a-statefulset
apiVersion: v1
kind: Service
metadata:
name: dgraph-alpha
labels:
app: dgraph-alpha
spec:
ports:
- port: 7080
targetPort: 7080
name: grpc-alpha-int
clusterIP: None
# We want all pods in the StatefulSet to have their addresses published for
# the sake of the other Dgraph alpha pods even before they're ready, since they
# have to be able to talk to each other in order to become ready.
publishNotReadyAddresses: true
selector:
app: dgraph-alpha
---
# This StatefulSet runs 3 Dgraph Zero.
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: dgraph-zero
spec:
serviceName: "dgraph-zero"
replicas: 3
selector:
matchLabels:
app: dgraph-zero
template:
metadata:
labels:
app: dgraph-zero
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- dgraph-zero
topologyKey: kubernetes.io/hostname
containers:
- name: zero
image: dgraph/dgraph:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5080
name: grpc-zero
- containerPort: 6080
name: http-zero
volumeMounts:
- name: datadir
mountPath: /root/dgraph/zero
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
command:
- bash
- "-c"
- |
set -ex
[[ `hostname` =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
idx=$(($ordinal + 1))
if [[ $ordinal -eq 0 ]]; then
exec dgraph zero --my=$(hostname -f):5080 --raft="idx=$idx" --replicas 3
else
exec dgraph zero --my=$(hostname -f):5080 --peer dgraph-zero-0.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080 --raft="idx=$idx" --replicas 3
fi
livenessProbe:
httpGet:
path: /health
port: 6080
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
successThreshold: 1
readinessProbe:
httpGet:
path: /root/dgraph/health
port: 6080
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
successThreshold: 1
terminationGracePeriodSeconds: 60
volumes:
- name: datadir
persistentVolumeClaim:
claimName: datadir
updateStrategy:
type: RollingUpdate
volumeClaimTemplates:
- metadata:
name: datadir
annotations:
volume.alpha.kubernetes.io/storage-class: anything
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: 5Gi
---
# This StatefulSet runs 3 replicas of Dgraph Alpha.
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: dgraph-alpha
spec:
serviceName: "dgraph-alpha"
replicas: 3
selector:
matchLabels:
app: dgraph-alpha
template:
metadata:
labels:
app: dgraph-alpha
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- dgraph-alpha
topologyKey: kubernetes.io/hostname
# Initializing the Alphas:
#
# You may want to initialize the Alphas with data before starting, e.g.
# with data from the Dgraph Bulk Loader: https://dgraph.io/docs/deploy/#bulk-loader.
# You can accomplish by uncommenting this initContainers config. This
# starts a container with the same /dgraph volume used by Alpha and runs
# before Alpha starts.
#
# You can copy your local p directory to the pod's /dgraph/p directory
# with this command:
#
# kubectl cp path/to/p dgraph-alpha-0:/dgraph/ -c init-alpha
# (repeat for each alpha pod)
#
# When you're finished initializing each Alpha data directory, you can signal
# it to terminate successfully by creating a /dgraph/doneinit file:
#
# kubectl exec dgraph-alpha-0 -c init-alpha touch /dgraph/doneinit
#
# Note that pod restarts cause re-execution of Init Containers. Since
# /dgraph is persisted across pod restarts, the Init Container will exit
# automatically when /dgraph/doneinit is present and proceed with starting
# the Alpha process.
#
# Tip: StatefulSet pods can start in parallel by configuring
# .spec.podManagementPolicy to Parallel:
#
# https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#deployment-and-scaling-guarantees
#
# initContainers:
# - name: init-alpha
# image: dgraph/dgraph:latest
# command:
# - bash
# - "-c"
# - |
# trap "exit" SIGINT SIGTERM
# echo "Write to /dgraph/doneinit when ready."
# until [ -f /dgraph/doneinit ]; do sleep 2; done
# volumeMounts:
# - name: datadir
# mountPath: /dgraph
containers:
- name: alpha
image: dgraph/dgraph:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 7080
name: grpc-alpha-int
- containerPort: 8080
name: http-alpha
- containerPort: 9080
name: grpc-alpha
volumeMounts:
- name: datadir
mountPath: /dgraph
env:
# This should be the same namespace as the dgraph-zero
# StatefulSet to resolve a Dgraph Zero's DNS name for
# Alpha's --zero flag.
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
# dgraph versions earlier than v1.2.3 and v20.03.0 can only support one zero:
# `dgraph alpha --zero dgraph-zero-0.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080`
# dgraph-alpha versions greater than or equal to v1.2.3 or v20.03.1 can support
# a comma-separated list of zeros. The value below supports 3 zeros
# (set according to number of replicas)
command:
- bash
- "-c"
- |
set -ex
dgraph alpha --my=$(hostname -f):7080 --zero dgraph-zero-0.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080,dgraph-zero-1.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080,dgraph-zero-2.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080
livenessProbe:
httpGet:
path: /health?live=1
port: 8080
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
successThreshold: 1
readinessProbe:
httpGet:
path: /root/dgraph/health
port: 8080
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
successThreshold: 1
terminationGracePeriodSeconds: 600
volumes:
- name: datadir
persistentVolumeClaim:
claimName: datadir
updateStrategy:
type: RollingUpdate
volumeClaimTemplates:
- metadata:
name: datadir
annotations:
volume.alpha.kubernetes.io/storage-class: anything
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: 5Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: dgraph-ratel
labels:
app: dgraph-ratel
spec:
selector:
matchLabels:
app: dgraph-ratel
template:
metadata:
labels:
app: dgraph-ratel
spec:
containers:
- name: ratel
image: dgraph/ratel:latest
ports:
- containerPort: 8000
I applied them all, without, apparently, any issues:
root@k8s-eu-1-control-plane-node-1:~# kubectl apply -f DgraphConfig/dgraph-local-storage-class.yaml
storageclass.storage.k8s.io/local-storage-dgraph created
root@k8s-eu-1-control-plane-node-1:~# kubectl apply -f DgraphConfig/dgraph-pv-alpha-control-plane-node-1.yaml
persistentvolume/dgraph-pv-alpha-control-plane-node-1 created
root@k8s-eu-1-control-plane-node-1:~# kubectl apply -f ./DgraphConfig/dgraph-pv-alpha-control-plane-node-2.yaml
persistentvolume/dgraph-pv-alpha-control-plane-node-2 created
root@k8s-eu-1-control-plane-node-1:~# kubectl apply -f ./DgraphConfig/dgraph-pv-alpha-control-plane-node-3.yaml
persistentvolume/dgraph-pv-alpha-control-plane-node-3 created
root@k8s-eu-1-control-plane-node-1:~# kubectl apply -f ./DgraphConfig/dgraph-pv-zero-control-worker-node-1.yaml
persistentvolume/dgraph-pv-zero-worker-node-1 created
root@k8s-eu-1-control-plane-node-1:~# kubectl apply -f ./DgraphConfig/dgraph-pv-zero-control-worker-node-2.yaml
persistentvolume/dgraph-pv-zero-worker-node-2 created
root@k8s-eu-1-control-plane-node-1:~# kubectl apply -f ./DgraphConfig/dgraph-pv-zero-control-worker-node-3.yaml
persistentvolume/dgraph-pv-zero-worker-node-3 created
root@k8s-eu-1-control-plane-node-1:~# kubectl apply -f ./DgraphConfig/dgraph-ha/dgraph-ha.yaml
service/dgraph-zero-public created
service/dgraph-alpha-public created
service/dgraph-ratel-public created
service/dgraph-zero created
service/dgraph-alpha created
statefulset.apps/dgraph-zero created
statefulset.apps/dgraph-alpha created
deployment.apps/dgraph-ratel created
But the dgraph resources are not actually deployed and working:
root@k8s-eu-1-control-plane-node-1:~# kubectl get deployments
NAME READY UP-TO-DATE AVAILABLE AGE
dgraph-ratel 1/1 1 1 121m
root@k8s-eu-1-control-plane-node-1:~# kubectl get storageclass
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
local-storage-dgraph kubernetes.io/no-provisioner Delete WaitForFirstConsumer false 154m
local-storage-nats-jetstream kubernetes.io/no-provisioner Delete WaitForFirstConsumer false 119d
local-storage-risingwave (default) kubernetes.io/no-provisioner Delete WaitForFirstConsumer false 116d
root@k8s-eu-1-control-plane-node-1:~# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
dgraph-pv-alpha-control-plane-node-1 350Gi RWO Delete Available local-storage-dgraph 141m
dgraph-pv-alpha-control-plane-node-2 350Gi RWO Delete Available local-storage-dgraph 132m
dgraph-pv-alpha-control-plane-node-3 350Gi RWO Delete Available local-storage-dgraph 132m
dgraph-pv-zero-worker-node-1 200Gi RWO Delete Available local-storage-dgraph 133m
dgraph-pv-zero-worker-node-2 200Gi RWO Delete Available local-storage-dgraph 131m
dgraph-pv-zero-worker-node-3 200Gi RWO Delete Available local-storage-dgraph 131m
nats-pv-jetstream-worker-1 10Gi RWO Delete Bound nats/nats-js-nats-1 local-storage-nats-jetstream 119d
nats-pv-jetstream-worker-2 10Gi RWO Delete Bound nats/nats-js-nats-0 local-storage-nats-jetstream 119d
nats-pv-jetstream-worker-3 10Gi RWO Delete Bound nats/nats-js-nats-2 local-storage-nats-jetstream 119d
risingwave-pv-compactor-worker-3 95Gi RWO Delete Bound risingwave/risingwave-minio local-storage-risingwave 116d
risingwave-pv-compute-control-plane-node-1 100Gi RWO Delete Bound default/data-etcdcharts-0 local-storage-risingwave 116d
risingwave-pv-compute-control-plane-node-2 100Gi RWO Delete Bound default/data-etcd-0 local-storage-risingwave 116d
risingwave-pv-compute-control-plane-node-3 100Gi RWO Delete Bound default/data-etcd-2 local-storage-risingwave 116d
risingwave-pv-frontend-worker-3 5Gi RWO Delete Bound default/datadir-dgraph-alpha-0 local-storage-risingwave 116d
risingwave-pv-meta-worker-1 100Gi RWO Delete Bound risingwave/data-risingwave-etcd-0 local-storage-risingwave 116d
risingwave-pv-meta-worker-2 100Gi RWO Delete Bound default/data-etcd-1 local-storage-risingwave 116d
root@k8s-eu-1-control-plane-node-1:~#
root@k8s-eu-1-control-plane-node-1:~# kubectl get pods -n dgraph
NAME READY STATUS RESTARTS AGE
dgraph-dgraph-alpha-0 0/1 Pending 0 21h
dgraph-dgraph-alpha-1 0/1 Pending 0 21h
dgraph-dgraph-alpha-2 0/1 Pending 0 21h
dgraph-dgraph-zero-0 0/1 Pending 0 21h
dgraph-dgraph-zero-1 0/1 Pending 0 21h
dgraph-dgraph-zero-2 0/1 Pending 0 21h
What am I missing and / or doing wrong?
How to make it work?