Output:
statefulset.apps/zammad created
Error from server (BadRequest): error when creating "STDIN": StatefulSet in version "v1" cannot be handled as a StatefulSet: json: cannot unmarshal bool into Go struct field EnvVar.spec.template.spec.containers.env.value of type string
Error from server (BadRequest): error when creating "STDIN": StatefulSet in version "v1" cannot be handled as a StatefulSet: json: cannot unmarshal bool into Go struct field EnvVar.spec.template.spec.containers.env.value of type string
Error from server (BadRequest): error when creating "STDIN": StatefulSet in version "v1" cannot be handled as a StatefulSet: json: cannot unmarshal bool into Go struct field EnvVar.spec.template.spec.containers.env.value of type string


bash

/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad scale sts zammad --replicas=0 || true; sudo kubectl --kubeconfig=<redacted> -n zammad delete sts zammad --wait=true || true; sudo kubectl --kubeconfig=<redacted> -n zammad get sts,pods -o wide'"


Output:
statefulset.apps/zammad scaled
statefulset.apps "zammad" deleted
NAME           READY   STATUS        RESTARTS   AGE   IP       NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-0   0/4     Terminating   0          12s   <none>   mbrkanov-mini-it13   <none>           <none>


/bin/bash -lc "for i in 1 2 3 4 5 6 7 8 9 10; do
  ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad get pod zammad-0 --ignore-not-found'
  sleep 3
done
python3 - <<'PY' > /tmp/zammad-longhorn-sts-raw.yaml
from pathlib import Path
text=Path('/home/node/.openclaw/workspace/repos/platform-gitops/tenants/pidoka/prod/zammad/app.yaml').read_text()
for doc in text.split('\\n---'):
    d=doc.strip('\\n')
    if 'kind: StatefulSet' not in d:
        continue
    if any(f'\\n  name: {n}\\n' in '\\n'+d+'\\n' for n in ['zammad','zammad-postgresql','zammad-redis-master','zammad-elasticsearch-master']):
        print('---')
        print(d)
PY
rg -n \"kind: StatefulSet|name: zammad"'$|name: zammad-postgresql|name: zammad-redis-master|name: zammad-elasticsearch-master|value: (false|no)$" /tmp/zammad-longhorn-sts-raw.yaml'


Output:
4:kind: StatefulSet
6:  name: zammad-elasticsearch-master
148:kind: StatefulSet
150:  name: zammad-postgresql
171:      name: zammad-postgresql
221:                  name: zammad-postgresql
298:kind: StatefulSet
300:  name: zammad-redis-master
444:kind: StatefulSet
446:  name: zammad
449:    app.kubernetes.io/name: zammad
458:      app.kubernetes.io/name: zammad
464:        app.kubernetes.io/name: zammad
508:                  name: zammad-postgresql-pass
546:                  name: zammad-postgresql-pass
584:                  name: zammad-postgresql-pass
689:                  name: zammad-postgresql-pass
748:                  name: zammad-postgresql-pass
791:                  name: zammad-postgresql-pass


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'set -euo pipefail
K=\"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml\"
"'$K apply -f -
$K -n zammad scale deploy zammad-memcached --replicas=1
$K -n zammad get sts,deploy,pods -o wide
'"' < /tmp/zammad-longhorn-sts-raw.yaml"


Output:
statefulset.apps/zammad-elasticsearch-master created
statefulset.apps/zammad-postgresql created
statefulset.apps/zammad-redis-master created
statefulset.apps/zammad created
deployment.apps/zammad-memcached scaled
NAME                                           READY   AGE   CONTAINERS      IMAGES
statefulset.apps/zammad-elasticsearch-master   0/1     1s    elasticsearch   docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     1s    postgresql      docker.io/bitnami/postgresql:15.3.0-debian-11-r17
statefulset.apps/zammad-redis-master           0/1     1s    redis           docker.io/bitnami/redis:7.0.11-debian-11-r20

NAME                               READY   UP-TO-DATE   AVAILABLE   AGE      CONTAINERS   IMAGES                                            SELECTOR
deployment.apps/zammad-memcached   0/1     1            0           2y169d   memcached    docker.io/bitnami/memcached:1.6.21-debian-11-r4   app.kubernetes.io/instance=zammad,app.kubernetes.io/name=memcached

NAME                                   READY   STATUS              RESTARTS   AGE   IP       NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-elasticsearch-master-0      0/1     Init:0/1            0          1s    <none>   mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                0/1     ContainerCreating   0          1s    <none>   mini-it13-02         <none>           <none>
pod/zammad-redis-master-0              0/1     ContainerCreating   0          1s    <none>   mini-it13-02         <none>           <none>
pod/zammad-memcached-95c5bfd6b-rm9xq   0/1     ContainerCreating   0          0s    <none>   mini-it13-02         <none>           <none>


/bin/bash -lc 'for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
  echo "--- pod poll $i"
  ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 '"'sudo kubectl --kubeconfig=<redacted> -n zammad get sts,pods -o wide'
  sleep 10
done"


/bin/bash -lc "ps -eo pid,ppid,stat,cmd | rg 'pod poll|zammad get sts,pods|for i in 1 2 3 4 5 6 7 8 9 10 11 12'"


Output:
5151      33 Ss   /bin/bash -c for i in 1 2 3 4 5 6 7 8 9 10 11 12; do   echo "--- pod poll $i"   ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad get sts,pods -o wide'   sleep 10 done
   5188      33 Ss   /bin/bash -c ps -eo pid,ppid,stat,cmd | rg 'pod poll|zammad get sts,pods|for i in 1 2 3 4 5 6 7 8 9 10 11 12'
   5190    5188 S    rg pod poll|zammad get sts,pods|for i in 1 2 3 4 5 6 7 8 9 10 11 12


Output:
NAME                                           READY   AGE   CONTAINERS      IMAGES
statefulset.apps/zammad-elasticsearch-master   0/1     14s   elasticsearch   docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     14s   postgresql      docker.io/bitnami/postgresql:15.3.0-debian-11-r17
statefulset.apps/zammad-redis-master           0/1     14s   redis           docker.io/bitnami/redis:7.0.11-debian-11-r20

NAME                                   READY   STATUS              RESTARTS   AGE   IP            NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-elasticsearch-master-0      0/1     Init:0/1            0          14s   <none>        mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                0/1     ContainerCreating   0          14s   <none>        mini-it13-02         <none>           <none>
pod/zammad-redis-master-0              0/1     ContainerCreating   0          14s   <none>        mini-it13-02         <none>           <none>
pod/zammad-memcached-95c5bfd6b-rm9xq   0/1     ErrImagePull        0          13s   10.42.1.253   mini-it13-02         <none>           <none>
--- pod poll 2
NAME                                           READY   AGE   CONTAINERS      IMAGES
statefulset.apps/zammad-elasticsearch-master   0/1     25s   elasticsearch   docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     25s   postgresql      docker.io/bitnami/postgresql:15.3.0-debian-11-r17
statefulset.apps/zammad-redis-master           0/1     25s   redis           docker.io/bitnami/redis:7.0.11-debian-11-r20

NAME                                   READY   STATUS             RESTARTS   AGE   IP            NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-memcached-95c5bfd6b-rm9xq   0/1     ImagePullBackOff   0          24s   10.42.1.253   mini-it13-02         <none>           <none>
pod/zammad-redis-master-0              0/1     ErrImagePull       0          25s   10.42.1.254   mini-it13-02         <none>           <none>
pod/zammad-postgresql-0                0/1     ErrImagePull       0          25s   10.42.1.2     mini-it13-02         <none>           <none>
pod/zammad-elasticsearch-master-0      0/1     Running            0          25s   10.42.0.78    mbrkanov-mini-it13   <none>           <none>
--- pod poll 3
NAME                                           READY   AGE   CONTAINERS      IMAGES
statefulset.apps/zammad-elasticsearch-master   0/1     35s   elasticsearch   docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     35s   postgresql      docker.io/bitnami/postgresql:15.3.0-debian-11-r17
statefulset.apps/zammad-redis-master           0/1     35s   redis           docker.io/bitnami/redis:7.0.11-debian-11-r20

NAME                                   READY   STATUS             RESTARTS   AGE   IP            NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-elasticsearch-master-0      0/1     Running            0          35s   10.42.0.78    mbrkanov-mini-it13   <none>           <none>
pod/zammad-redis-master-0              0/1     ImagePullBackOff   0          35s   10.42.1.254   mini-it13-02         <none>           <none>
pod/zammad-memcached-95c5bfd6b-rm9xq   0/1     ErrImagePull       0          34s   10.42.1.253   mini-it13-02         <none>           <none>
pod/zammad-postgresql-0                0/1     ImagePullBackOff   0          35s   10.42.1.2     mini-it13-02         <none>           <none>
--- pod poll 4
NAME                                           READY   AGE   CONTAINERS      IMAGES
statefulset.apps/zammad-elasticsearch-master   0/1     46s   elasticsearch   docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     46s   postgresql      docker.io/bitnami/postgresql:15.3.0-debian-11-r17
statefulset.apps/zammad-redis-master           0/1     46s   redis           docker.io/bitnami/redis:7.0.11-debian-11-r20

NAME                                   READY   STATUS             RESTARTS   AGE   IP            NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-elasticsearch-master-0      0/1     Running            0          46s   10.42.0.78    mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                0/1     ErrImagePull       0          46s   10.42.1.2     mini-it13-02         <none>           <none>
pod/zammad-memcached-95c5bfd6b-rm9xq   0/1     ImagePullBackOff   0          45s   10.42.1.253   mini-it13-02         <none>           <none>
pod/zammad-redis-master-0              0/1     ErrImagePull       0          46s   10.42.1.254   mini-it13-02         <none>           <none>
--- pod poll 5
NAME                                           READY   AGE   CONTAINERS      IMAGES
statefulset.apps/zammad-elasticsearch-master   0/1     56s   elasticsearch   docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     56s   postgresql      docker.io/bitnami/postgresql:15.3.0-debian-11-r17
statefulset.apps/zammad-redis-master           0/1     56s   redis           docker.io/bitnami/redis:7.0.11-debian-11-r20

NAME                                   READY   STATUS             RESTARTS   AGE   IP            NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-elasticsearch-master-0      0/1     Running            0          56s   10.42.0.78    mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                0/1     ImagePullBackOff   0          56s   10.42.1.2     mini-it13-02         <none>           <none>
pod/zammad-memcached-95c5bfd6b-rm9xq   0/1     ErrImagePull       0          55s   10.42.1.253   mini-it13-02         <none>           <none>
pod/zammad-redis-master-0              0/1     ImagePullBackOff   0          56s   10.42.1.254   mini-it13-02         <none>           <none>
--- pod poll 6
NAME                                           READY   AGE   CONTAINERS      IMAGES
statefulset.apps/zammad-elasticsearch-master   0/1     67s   elasticsearch   docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     67s   postgresql      docker.io/bitnami/postgresql:15.3.0-debian-11-r17
statefulset.apps/zammad-redis-master           0/1     67s   redis           docker.io/bitnami/redis:7.0.11-debian-11-r20

NAME                                   READY   STATUS             RESTARTS   AGE   IP            NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-elasticsearch-master-0      0/1     Running            0          67s   10.42.0.78    mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                0/1     ImagePullBackOff   0          67s   10.42.1.2     mini-it13-02         <none>           <none>
pod/zammad-redis-master-0              0/1     ImagePullBackOff   0          67s   10.42.1.254   mini-it13-02         <none>           <none>
pod/zammad-memcached-95c5bfd6b-rm9xq   0/1     ImagePullBackOff   0          66s   10.42.1.253   mini-it13-02         <none>           <none>


/bin/bash -lc "python3 - <<'PY' > /tmp/zammad-sts-zammad-ns.yaml
from pathlib import Path
text=Path('/tmp/zammad-longhorn-desired.yaml').read_text()
for doc in text.split('\\n---'):
    d=doc.strip('\\n')
    if 'kind: StatefulSet' in d and '\\n  name: zammad\\n' in '\\n'+d+'\\n':
        print('---')
        print(d)
PY
ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'set -euo pipefail
K=\"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml\"
"'$K -n default delete sts zammad --ignore-not-found --wait=true
$K apply -f -
$K -n zammad get sts zammad -o wide
'"' < /tmp/zammad-sts-zammad-ns.yaml"


Output:
statefulset.apps "zammad" deleted
statefulset.apps/zammad created
NAME     READY   AGE   CONTAINERS                                                          IMAGES
zammad   0/1     0s    zammad-nginx,zammad-railsserver,zammad-scheduler,zammad-websocket   ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad get pods -o wide'"


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'for p in zammad-postgresql-0 zammad-redis-master-0 zammad-memcached-95c5bfd6b-rm9xq zammad-0; do echo ==== "'$p; sudo kubectl --kubeconfig=<redacted> -n zammad describe pod $p 2>/dev/null | tail -n 25 || true; done'"'"


Output:
NAME                               READY   STATUS             RESTARTS   AGE    IP            NODE                 NOMINATED NODE   READINESS GATES
zammad-elasticsearch-master-0      0/1     Running            0          105s   10.42.0.78    mbrkanov-mini-it13   <none>           <none>
zammad-memcached-95c5bfd6b-rm9xq   0/1     ImagePullBackOff   0          104s   10.42.1.253   mini-it13-02         <none>           <none>
zammad-postgresql-0                0/1     ImagePullBackOff   0          105s   10.42.1.2     mini-it13-02         <none>           <none>
zammad-redis-master-0              0/1     ImagePullBackOff   0          105s   10.42.1.254   mini-it13-02         <none>           <none>
zammad-0                           0/4     Init:0/4           0          13s    <none>        mbrkanov-mini-it13   <none>           <none>


Output:
==== zammad-postgresql-0
    SizeLimit:  <unset>
  data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  zammad-postgresql-data-longhorn
    ReadOnly:   false
  kube-api-access-4wqkg:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason                  Age                From                     Message
  ----     ------                  ----               ----                     -------
  Normal   Scheduled               104s               default-scheduler        Successfully assigned zammad/zammad-postgresql-0 to mini-it13-02
  Normal   SuccessfulAttachVolume  94s                attachdetach-controller  AttachVolume.Attach succeeded for volume "pvc-e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f"
  Normal   Pulling                 51s (x3 over 90s)  kubelet                  Pulling image "docker.io/bitnami/postgresql:15.3.0-debian-11-r17"
  Warning  Failed                  51s (x3 over 89s)  kubelet                  Failed to pull image "docker.io/bitnami/postgresql:15.3.0-debian-11-r17": rpc error: code = NotFound desc = failed to pull and unpack image "docker.io/bitnami/postgresql:15.3.0-debian-11-r17": failed to resolve reference "docker.io/bitnami/postgresql:15.3.0-debian-11-r17": docker.io/bitnami/postgresql:15.3.0-debian-11-r17: not found
  Warning  Failed                  51s (x3 over 89s)  kubelet                  Error: ErrImagePull
  Normal   BackOff                 12s (x5 over 88s)  kubelet                  Back-off pulling image "docker.io/bitnami/postgresql:15.3.0-debian-11-r17"
  Warning  Failed                  12s (x5 over 88s)  kubelet                  Error: ImagePullBackOff
==== zammad-redis-master-0
    SizeLimit:  <unset>
  redis-data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  zammad-redis-data-longhorn
    ReadOnly:   false
  kube-api-access-czw7v:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason                  Age                From                     Message
  ----     ------                  ----               ----                     -------
  Normal   Scheduled               104s               default-scheduler        Successfully assigned zammad/zammad-redis-master-0 to mini-it13-02
  Normal   SuccessfulAttachVolume  94s                attachdetach-controller  AttachVolume.Attach succeeded for volume "pvc-61d812f1-d3f3-4d63-a194-782128ae536f"
  Normal   BackOff                 18s (x4 over 89s)  kubelet                  Back-off pulling image "docker.io/bitnami/redis:7.0.11-debian-11-r20"
  Warning  Failed                  18s (x4 over 89s)  kubelet                  Error: ImagePullBackOff
  Normal   Pulling                 3s (x4 over 90s)   kubelet                  Pulling image "docker.io/bitnami/redis:7.0.11-debian-11-r20"
  Warning  Failed                  3s (x4 over 89s)   kubelet                  Failed to pull image "docker.io/bitnami/redis:7.0.11-debian-11-r20": rpc error: code = NotFound desc = failed to pull and unpack image "docker.io/bitnami/redis:7.0.11-debian-11-r20": failed to resolve reference "docker.io/bitnami/redis:7.0.11-debian-11-r20": docker.io/bitnami/redis:7.0.11-debian-11-r20: not found
  Warning  Failed                  3s (x4 over 89s)   kubelet                  Error: ErrImagePull
==== zammad-memcached-95c5bfd6b-rm9xq
  PodScheduled                True
Volumes:
  tmp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:
    SizeLimit:  <unset>
  kube-api-access-669qk:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age                 From               Message
  ----     ------     ----                ----               -------
  Normal   Scheduled  104s                default-scheduler  Successfully assigned zammad/zammad-memcached-95c5bfd6b-rm9xq to mini-it13-02
  Normal   BackOff    25s (x5 over 102s)  kubelet            Back-off pulling image "docker.io/bitnami/memcached:1.6.21-debian-11-r4"
  Warning  Failed     25s (x5 over 102s)  kubelet            Error: ImagePullBackOff
  Normal   Pulling    10s (x4 over 104s)  kubelet            Pulling image "docker.io/bitnami/memcached:1.6.21-debian-11-r4"
  Warning  Failed     10s (x4 over 103s)  kubelet            Failed to pull image "docker.io/bitnami/memcached:1.6.21-debian-11-r4": rpc error: code = NotFound desc = failed to pull and unpack image "docker.io/bitnami/memcached:1.6.21-debian-11-r4": failed to resolve reference "docker.io/bitnami/memcached:1.6.21-debian-11-r4": docker.io/bitnami/memcached:1.6.21-debian-11-r4: not found
  Warning  Failed     10s (x4 over 103s)  kubelet            Error: ErrImagePull
==== zammad-0
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:
    SizeLimit:  100Mi
  zammad-var:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  zammad-var-longhorn
    ReadOnly:   false
  kube-api-access-gxq7r:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type    Reason                  Age   From                     Message
  ----    ------                  ----  ----                     -------
  Normal  Scheduled               13s   default-scheduler        Successfully assigned zammad/zammad-0 to mbrkanov-mini-it13
  Normal  SuccessfulAttachVolume  3s    attachdetach-controller  AttachVolume.Attach succeeded for volume "pvc-ed7727c1-57f6-4ed0-8dc2-3f7854de1339"
  Normal  Pulled                  0s    kubelet                  Container image "alpine:3.18.4" already present on machine
  Normal  Created                 0s    kubelet                  Created container zammad-volume-permissions
  Normal  Started                 0s    kubelet                  Started container zammad-volume-permissions


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'set -euo pipefail
K=\"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml\"
PATCH='\\''{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\"mbrkanov-mini-it13\"}}}}}'\\''
"'$K -n zammad patch sts zammad zammad-postgresql zammad-redis-master zammad-elasticsearch-master --type merge -p "$PATCH"
$K -n zammad patch deploy zammad-memcached --type merge -p "$PATCH"
$K -n zammad delete pod zammad-postgresql-0 zammad-redis-master-0 zammad-memcached-95c5bfd6b-rm9xq --ignore-not-found
$K -n zammad get pods -o wide
'"'"


Output:
statefulset.apps/zammad patched
statefulset.apps/zammad-postgresql patched
statefulset.apps/zammad-redis-master patched
statefulset.apps/zammad-elasticsearch-master patched
deployment.apps/zammad-memcached patched
pod "zammad-postgresql-0" deleted
pod "zammad-redis-master-0" deleted
pod "zammad-memcached-95c5bfd6b-rm9xq" deleted
NAME                                READY   STATUS              RESTARTS      AGE    IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-elasticsearch-master-0       1/1     Terminating         0             2m5s   10.42.0.78   mbrkanov-mini-it13   <none>           <none>
zammad-memcached-95c5bfd6b-h6m6v    0/1     ContainerCreating   0             2s     <none>       mini-it13-02         <none>           <none>
zammad-postgresql-0                 0/1     ContainerCreating   0             2s     <none>       mbrkanov-mini-it13   <none>           <none>
zammad-memcached-68ff6f5577-nrs4k   0/1     Running             0             2s     10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:1/4            2 (13s ago)   33s    10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               0/1     ContainerCreating   0             1s     <none>       mbrkanov-mini-it13   <none>           <none>


/bin/bash -lc 'for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
  echo "--- recover poll $i"
  ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 '"'sudo kubectl --kubeconfig=<redacted> -n zammad get pods -o wide'
  sleep 10
done"


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad describe pod zammad-postgresql-0 | tail -n 80'"


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad logs zammad-postgresql-0 --previous --tail=120 || sudo kubectl --kubeconfig=<redacted> -n zammad logs zammad-postgresql-0 --tail=120 || true'"


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad logs zammad-0 -c postgresql-init --previous --tail=80 || true'"


Output:
postgresql 22:31:20.41
postgresql 22:31:20.41 Welcome to the Bitnami postgresql container
postgresql 22:31:20.41 Subscribe to project updates by watching https://github.com/bitnami/containers
postgresql 22:31:20.41 Submit issues and feature requests at https://github.com/bitnami/containers/issues
postgresql 22:31:20.41
postgresql 22:31:20.44 INFO  ==> ** Starting PostgreSQL setup **
postgresql 22:31:20.46 INFO  ==> Validating settings in POSTGRESQL_* env vars..
postgresql 22:31:20.47 INFO  ==> Loading custom pre-init scripts...
postgresql 22:31:20.48 INFO  ==> Initializing PostgreSQL database...
postgresql 22:31:20.53 INFO  ==> pg_hba.conf file not detected. Generating it...
postgresql 22:31:20.54 INFO  ==> Generating local authentication configuration
postgresql 22:31:20.57 INFO  ==> Deploying PostgreSQL with persisted data...
postgresql 22:31:20.59 INFO  ==> Configuring replication parameters
postgresql 22:31:20.63 INFO  ==> Configuring fsync
postgresql 22:31:20.65 INFO  ==> Configuring synchronous_replication
postgresql 22:31:20.69 INFO  ==> Loading custom scripts...
postgresql 22:31:20.70 INFO  ==> Enabling remote connections
postgresql 22:31:20.72 INFO  ==> ** PostgreSQL setup finished! **

postgresql 22:31:20.74 INFO  ==> ** Starting PostgreSQL **
2026-06-17 22:31:20.761 GMT [1] FATAL:  "/bitnami/postgresql/data" is not a valid data directory
2026-06-17 22:31:20.761 GMT [1] DETAIL:  File "/bitnami/postgresql/data/PG_VERSION" does not contain valid data.
2026-06-17 22:31:20.761 GMT [1] HINT:  You might need to initdb.


Output:
There was an error trying to connect to Redis via redis://:zammad@zammad-redis-master:6379.
Please make sure Redis is available.
#<Redis::CannotConnectError: Error connecting to Redis on zammad-redis-master:6379 (Errno::ECONNREFUSED)>


Output:
Controlled By:  StatefulSet/zammad-postgresql
Containers:
  postgresql:
    Container ID:   containerd://0d2b7cac4944a73293b82e903ccfba2614e1f32bfe4a01ef348bcfb924831ea4
    Image:          docker.io/bitnami/postgresql:15.3.0-debian-11-r17
    Image ID:       docker.io/bitnami/postgresql@sha256:7f6b00d6b84b0f3115c15aa06694c4dc80e4999ef7ba2b30ec31fc4ded904f13
    Port:           5432/TCP
    Host Port:      0/TCP
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    1
      Started:      Thu, 18 Jun 2026 00:31:20 +0200
      Finished:     Thu, 18 Jun 2026 00:31:20 +0200
    Ready:          False
    Restart Count:  2
    Requests:
      cpu:      250m
      memory:   256Mi
    Liveness:   exec [/bin/sh -c exec pg_isready -U "zammad" -d "dbname=zammad_production" -h 127.0.0.1 -p 5432] delay=30s timeout=5s period=10s #success=1 #failure=6
    Readiness:  exec [/bin/sh -c -e exec pg_isready -U "zammad" -d "dbname=zammad_production" -h 127.0.0.1 -p 5432
[ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ]
] delay=5s timeout=5s period=10s #success=1 #failure=6
    Environment:
      BITNAMI_DEBUG:                        false
      POSTGRESQL_PORT_NUMBER:               5432
      POSTGRESQL_VOLUME_DIR:                /bitnami/postgresql
      PGDATA:                               /bitnami/postgresql/data
      POSTGRES_USER:                        zammad
      POSTGRES_PASSWORD:                    <set to the key 'password' in secret 'zammad-postgresql'>  Optional: false
      POSTGRES_POSTGRES_PASSWORD:           zammad
      POSTGRES_DATABASE:                    zammad_production
      POSTGRESQL_ENABLE_LDAP:               no
      POSTGRESQL_ENABLE_TLS:                no
      POSTGRESQL_LOG_HOSTNAME:              false
      POSTGRESQL_LOG_CONNECTIONS:           false
      POSTGRESQL_LOG_DISCONNECTIONS:        false
      POSTGRESQL_PGAUDIT_LOG_CATALOG:       off
      POSTGRESQL_CLIENT_MIN_MESSAGES:       error
      POSTGRESQL_SHARED_PRELOAD_LIBRARIES:  pgaudit
    Mounts:
      /bitnami/postgresql from data (rw)
      /dev/shm from dshm (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-k7m4b (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  dshm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  zammad-postgresql-data-longhorn
    ReadOnly:   false
  kube-api-access-k7m4b:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/hostname=<redacted>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason                  Age                From                     Message
  ----     ------                  ----               ----                     -------
  Normal   Scheduled               53s                default-scheduler        Successfully assigned zammad/zammad-postgresql-0 to mbrkanov-mini-it13
  Warning  FailedAttachVolume      53s                attachdetach-controller  Multi-Attach error for volume "pvc-e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f" Volume is already exclusively attached to one node and can't be attached to another
  Normal   SuccessfulAttachVolume  39s                attachdetach-controller  AttachVolume.Attach succeeded for volume "pvc-e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f"
  Normal   Pulled                  12s (x3 over 36s)  kubelet                  Container image "docker.io/bitnami/postgresql:15.3.0-debian-11-r17" already present on machine
  Normal   Created                 12s (x3 over 36s)  kubelet                  Created container postgresql
  Normal   Started                 12s (x3 over 36s)  kubelet                  Started container postgresql
  Warning  BackOff                 6s (x5 over 33s)   kubelet                  Back-off restarting failed container postgresql in pod zammad-postgresql-0_zammad(1f729811-bb38-4347-b270-a76f6979da79)


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'set -euo pipefail
K=\"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml\"
"'$K -n zammad scale sts zammad-postgresql --replicas=0
for i in $(seq 1 30); do $K -n zammad get pod zammad-postgresql-0 --ignore-not-found | grep -q zammad-postgresql-0 || break; sleep 2; done
cat <<EOF | $K apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: zammad-inspect-pg-pvcs
  namespace: zammad
spec:
  restartPolicy: Never
  nodeSelector:
    kubernetes.io/hostname: mbrkanov-mini-it13
  containers:
    - name: inspect
      image: alpine:3.18.4
      imagePullPolicy: IfNotPresent
      command: ["sh", "-c", "echo OLD; find /old -maxdepth 3 -mindepth 1 -printf '"'%y %p %s\\\\n' | sort | head -80; echo NEW; find /new -maxdepth 3 -mindepth 1 -printf '%y %p %s\\\\n' | sort | head -80; echo OLD_PG_VERSION; ls -l /old/data/PG_VERSION /old/postgresql/data/PG_VERSION 2>/dev/null || true; echo NEW_PG_VERSION; ls -l /new/data/PG_VERSION /new/postgresql/data/PG_VERSION 2>/dev/null || true; sleep 5\"]
      volumeMounts:
        - name: old
          mountPath: /old
          readOnly: true
        - name: new
          mountPath: /new
          readOnly: true
  volumes:
    - name: old
      persistentVolumeClaim:
        claimName: data-zammad-postgresql-0
    - name: new
      persistentVolumeClaim:
        claimName: zammad-postgresql-data-longhorn
EOF
for i in "'$(seq 1 60); do phase=$($K -n zammad get pod zammad-inspect-pg-pvcs -o jsonpath='"'{.status.phase}' 2>/dev/null || true); [ \""'$phase" = Succeeded ] && break; [ "$phase" = Failed ] && break; sleep 2; done
$K -n zammad logs zammad-inspect-pg-pvcs
$K -n zammad delete pod zammad-inspect-pg-pvcs --wait=true
'"'"


Output:
statefulset.apps/zammad-postgresql scaled
pod/zammad-inspect-pg-pvcs created
OLD
find: unrecognized: -printf
BusyBox v1.36.1 (2023-07-27 17:12:24 UTC) multi-call binary.

Usage: find [-HL] [PATH]... [OPTIONS] [ACTIONS]

Search for files and perform actions on them.
First failed action stops processing of current file.
Defaults: PATH is current directory, action is '-print'

	-L,-follow	Follow symlinks
	-H		...on command line only
	-xdev		Don't descend directories on other filesystems
	-maxdepth N	Descend at most N levels. -maxdepth 0 applies
			actions to command line arguments only
	-mindepth N	Don't act on first N levels
	-depth		Act on directory *after* traversing it

Actions:
	( ACTIONS )	Group actions for -o / -a
	! ACT		Invert ACT's success/failure
	ACT1 [-a] ACT2	If ACT1 fails, stop, else do ACT2
	ACT1 -o ACT2	If ACT1 succeeds, stop, else do ACT2
			Note: -a has higher priority than -o
	-name PATTERN	Match file name (w/o directory name) to PATTERN
	-iname PATTERN	Case insensitive -name
	-path PATTERN	Match path to PATTERN
	-ipath PATTERN	Case insensitive -path
	-regex PATTERN	Match path to regex PATTERN
	-type X		File type is X (one of: f,d,l,b,c,s,p)
	-executable	File is executable
	-perm MASK	At least one mask bit (+MASK), all bits (-MASK),
			or exactly MASK bits are set in file's mode
	-mtime DAYS	mtime is greater than (+N), less than (-N),
			or exactly N days in the past
	-atime DAYS	atime +N/-N/N days in the past
	-ctime DAYS	ctime +N/-N/N days in the past
	-mmin MINS	mtime is greater than (+N), less than (-N),
			or exactly N minutes in the past
	-newer FILE	mtime is more recent than FILE's
	-inum N		File has inode number N
	-user NAME/ID	File is owned by given user
	-group NAME/ID	File is owned by given group
	-size N[bck]	File size is N (c:bytes,k:kbytes,b:512 bytes(def.))
			+/-N: file size is bigger/smaller than N
	-links N	Number of links is greater than (+N), less than (-N),
			or exactly N
	-empty		Match empty file/directory
	-prune		If current file is directory, don't descend into it
If none of the following actions is specified, -print is assumed
	-print		Print file name
	-print0		Print file name, NUL terminated
	-exec CMD ARG ;	Run CMD with all instances of {} replaced by
			file name. Fails if CMD exits with nonzero
	-exec CMD ARG + Run CMD with {} replaced by list of file names
	-delete		Delete current file/directory. Turns on -depth option
	-quit		Exit
sh: syntax error: unexpected "|"
pod "zammad-inspect-pg-pvcs" deleted


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'set -euo pipefail
K=\"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml\"
cat <<EOF | "'$K apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: zammad-inspect-pg-pvcs
  namespace: zammad
spec:
  restartPolicy: Never
  nodeSelector:
    kubernetes.io/hostname: mbrkanov-mini-it13
  containers:
    - name: inspect
      image: alpine:3.18.4
      imagePullPolicy: IfNotPresent
      command:
        - sh
        - -c
        - |
          set -eu
          for base in /old /new; do
            echo ==== $base
            ls -la $base | sed -n "1,40p"
            for d in data postgresql postgresql/data; do echo -- $base/$d; ls -la $base/$d 2>/dev/null | sed -n "1,25p" || true; done
            echo PG_VERSION_FILES
            for f in $base/data/PG_VERSION $base/postgresql/data/PG_VERSION; do [ -e "$f" ] && { ls -l "$f"; printf "content="; cat "$f"; echo; }; done
          done
      volumeMounts:
        - name: old
          mountPath: /old
          readOnly: true
        - name: new
          mountPath: /new
          readOnly: true
  volumes:
    - name: old
      persistentVolumeClaim:
        claimName: data-zammad-postgresql-0
    - name: new
      persistentVolumeClaim:
        claimName: zammad-postgresql-data-longhorn
EOF
for i in $(seq 1 60); do phase=$($K -n zammad get pod zammad-inspect-pg-pvcs -o jsonpath='"'{.status.phase}' 2>/dev/null || true); [ \""'$phase" = Succeeded ] && break; [ "$phase" = Failed ] && break; sleep 2; done
$K -n zammad logs zammad-inspect-pg-pvcs
$K -n zammad delete pod zammad-inspect-pg-pvcs --wait=true
'"'"


Output:
bash: line 3: base: unbound variable
error: no objects passed to apply


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'set -euo pipefail
K=\"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml\"
cat <<'\\''EOF'\\'' | "'$K apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: zammad-inspect-pg-pvcs
  namespace: zammad
spec:
  restartPolicy: Never
  nodeSelector:
    kubernetes.io/hostname: mbrkanov-mini-it13
  containers:
    - name: inspect
      image: alpine:3.18.4
      imagePullPolicy: IfNotPresent
      command:
        - sh
        - -c
        - |
          set -eu
          for base in /old /new; do
            echo ==== $base
            ls -la $base | sed -n "1,40p"
            for d in data postgresql postgresql/data; do echo -- $base/$d; ls -la $base/$d 2>/dev/null | sed -n "1,25p" || true; done
            echo PG_VERSION_FILES
            for f in $base/data/PG_VERSION $base/postgresql/data/PG_VERSION; do [ -e "$f" ] && { ls -l "$f"; printf "content="; cat "$f"; echo; }; done
          done
      volumeMounts:
        - name: old
          mountPath: /old
          readOnly: true
        - name: new
          mountPath: /new
          readOnly: true
  volumes:
    - name: old
      persistentVolumeClaim:
        claimName: data-zammad-postgresql-0
    - name: new
      persistentVolumeClaim:
        claimName: zammad-postgresql-data-longhorn
EOF
for i in $(seq 1 60); do phase=$($K -n zammad get pod zammad-inspect-pg-pvcs -o jsonpath='"'{.status.phase}' 2>/dev/null || true); [ \""'$phase" = Succeeded ] && break; [ "$phase" = Failed ] && break; sleep 2; done
$K -n zammad logs zammad-inspect-pg-pvcs
$K -n zammad delete pod zammad-inspect-pg-pvcs --wait=true
'"'"


Output:
NAME                                READY   STATUS              RESTARTS      AGE   IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-postgresql-0                 0/1     ContainerCreating   0             10s   <none>       mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               0/1     ContainerCreating   0             9s    <none>       mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:Error          2 (21s ago)   41s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Init:0/1            0             7s    <none>       mbrkanov-mini-it13   <none>           <none>
zammad-memcached-68ff6f5577-nrs4k   1/1     Running             0             10s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
--- recover poll 2
NAME                                READY   STATUS                  RESTARTS      AGE   IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             21s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running                 0             18s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:CrashLoopBackOff   2 (19s ago)   52s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               0/1     Running                 0             20s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-postgresql-0                 0/1     Error                   1 (2s ago)    21s   10.42.0.82   mbrkanov-mini-it13   <none>           <none>
--- recover poll 3
NAME                                READY   STATUS             RESTARTS      AGE   IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running            0             31s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running            0             28s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               0/1     Running            0             30s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-postgresql-0                 0/1     CrashLoopBackOff   1 (11s ago)   31s   10.42.0.82   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:Error         3 (29s ago)   62s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
--- recover poll 4
NAME                                READY   STATUS             RESTARTS      AGE   IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running            0             42s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running            0             39s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-postgresql-0                 0/1     CrashLoopBackOff   1 (22s ago)   42s   10.42.0.82   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:Error         3 (40s ago)   73s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running            0             41s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
--- recover poll 5
NAME                                READY   STATUS                  RESTARTS      AGE   IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             52s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running                 0             49s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running                 0             51s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:CrashLoopBackOff   3 (22s ago)   83s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-postgresql-0                 0/1     CrashLoopBackOff        2 (10s ago)   52s   10.42.0.82   mbrkanov-mini-it13   <none>           <none>
--- recover poll 6
NAME                                READY   STATUS                  RESTARTS      AGE   IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             63s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running                 0             60s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running                 0             62s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:CrashLoopBackOff   3 (33s ago)   94s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-postgresql-0                 0/1     CrashLoopBackOff        2 (21s ago)   63s   10.42.0.82   mbrkanov-mini-it13   <none>           <none>
--- recover poll 7
NAME                                READY   STATUS                  RESTARTS      AGE    IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             73s    10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running                 0             70s    10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running                 0             72s    10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:CrashLoopBackOff   3 (43s ago)   104s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-postgresql-0                 0/1     CrashLoopBackOff        3 (10s ago)   73s    10.42.0.82   mbrkanov-mini-it13   <none>           <none>
--- recover poll 8
NAME                                READY   STATUS              RESTARTS   AGE    IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running             0          84s    10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running             0          81s    10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running             0          83s    10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-inspect-pg-pvcs              0/1     ContainerCreating   0          8s     <none>       mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:1/4            4          115s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
--- recover poll 9
NAME                                READY   STATUS       RESTARTS   AGE    IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running      0          94s    10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running      0          91s    10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running      0          93s    10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:Error   4          2m5s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
--- recover poll 10
NAME                                READY   STATUS                  RESTARTS      AGE     IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             105s    10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       0/1     Running                 0             102s    10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running                 0             104s    10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:CrashLoopBackOff   4 (22s ago)   2m16s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
--- recover poll 11
NAME                                READY   STATUS                  RESTARTS      AGE     IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             115s    10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running                 0             114s    10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:CrashLoopBackOff   4 (32s ago)   2m26s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       1/1     Running                 0             112s    10.42.0.81   mbrkanov-mini-it13   <none>           <none>
--- recover poll 12
NAME                                READY   STATUS                  RESTARTS      AGE     IP           NODE                 NOMINATED NODE   READINESS GATES
zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             2m6s    10.42.0.80   mbrkanov-mini-it13   <none>           <none>
zammad-redis-master-0               1/1     Running                 0             2m5s    10.42.0.83   mbrkanov-mini-it13   <none>           <none>
zammad-0                            0/4     Init:CrashLoopBackOff   4 (43s ago)   2m37s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
zammad-elasticsearch-master-0       1/1     Running                 0             2m3s    10.42.0.81   mbrkanov-mini-it13   <none>           <none>
zammad-inspect-pg-pvcs              0/1     ContainerCreating       0             4s      <none>       mbrkanov-mini-it13   <none>           <none>


Output:
pod/zammad-inspect-pg-pvcs created
==== /old
total 12
drwxrwxrwx    3 root     root          4096 Dec 31  2023 .
drwxr-xr-x    1 root     root          4096 Jun 17 22:32 ..
drwx------   19 1001     root          4096 Jun 17 22:20 data
-- /old/data
total 92
drwx------   19 1001     root          4096 Jun 17 22:20 .
drwxrwxrwx    3 root     root          4096 Dec 31  2023 ..
-rw-------    1 1001     root             3 Dec 31  2023 PG_VERSION
drwx------    6 1001     root          4096 Dec 31  2023 base
drwx------    2 1001     root          4096 Jun  7 19:50 global
drwx------    2 1001     root          4096 Dec 31  2023 pg_commit_ts
drwx------    2 1001     root          4096 Dec 31  2023 pg_dynshmem
-rw-------    1 1001     root          1636 Dec 31  2023 pg_ident.conf
drwx------    4 1001     root          4096 Jun 17 22:20 pg_logical
drwx------    4 1001     root          4096 Dec 31  2023 pg_multixact
drwx------    2 1001     root          4096 Dec 31  2023 pg_notify
drwx------    2 1001     root          4096 Dec 31  2023 pg_replslot
drwx------    2 1001     root          4096 Dec 31  2023 pg_serial
drwx------    2 1001     root          4096 Dec 31  2023 pg_snapshots
drwx------    2 1001     root          4096 Jun 17 22:20 pg_stat
drwx------    2 1001     root          4096 Dec 31  2023 pg_stat_tmp
drwx------    2 1001     root          4096 Jun 17 06:04 pg_subtrans
drwx------    2 1001     root          4096 Dec 31  2023 pg_tblspc
drwx------    2 1001     root          4096 Dec 31  2023 pg_twophase
drwx------    3 1001     root          4096 Jun 17 17:05 pg_wal
drwx------    2 1001     root          4096 Jun 15 17:49 pg_xact
-rw-------    1 1001     root            88 Dec 31  2023 postgresql.auto.conf
-rw-------    1 1001     root           249 May  3 09:51 postmaster.opts
-- /old/postgresql
-- /old/postgresql/data
PG_VERSION_FILES
-rw-------    1 1001     root             3 Dec 31  2023 /old/data/PG_VERSION
content=15

==== /new
total 12
drwxrwsrwx    3 root     1001          4096 Jun 17 22:32 .
drwxr-xr-x    1 root     root          4096 Jun 17 22:32 ..
drwx--S---   19 1001     1001          4096 Jun 17 22:21 data
-- /new/data
total 88
drwx--S---   19 1001     1001          4096 Jun 17 22:21 .
drwxrwsrwx    3 root     1001          4096 Jun 17 22:32 ..
-rw-rw----    1 1001     1001             0 Dec 31  2023 PG_VERSION
drwxrws---    6 1001     1001          4096 Jun 17 22:21 base
drwxrws---    2 1001     1001          4096 Jun 17 22:21 global
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_commit_ts
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_dynshmem
-rw-rw----    1 1001     1001          1636 Dec 31  2023 pg_ident.conf
drwxrws---    4 1001     1001          4096 Jun 17 22:21 pg_logical
drwxrws---    4 1001     1001          4096 Jun 17 22:21 pg_multixact
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_notify
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_replslot
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_serial
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_snapshots
drwxrws---    2 1001     1001          4096 Jun 17 22:21 pg_stat
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_stat_tmp
drwxrws---    2 1001     1001          4096 Jun 17 22:21 pg_subtrans
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_tblspc
drwxrws---    2 1001     1001          4096 Dec 31  2023 pg_twophase
drwxrws---    3 1001     1001          4096 Jun 17 22:21 pg_wal
drwxrws---    2 1001     1001          4096 Jun 17 22:21 pg_xact
-rw-rw----    1 1001     1001            88 Dec 31  2023 postgresql.auto.conf
-rw-rw----    1 1001     1001           249 May  3 09:51 postmaster.opts
-- /new/postgresql
-- /new/postgresql/data
PG_VERSION_FILES
-rw-rw----    1 1001     1001             0 Dec 31  2023 /new/data/PG_VERSION
content=
pod "zammad-inspect-pg-pvcs" deleted


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'set -euo pipefail
K=\"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml\"
"'$K -n zammad scale sts zammad-postgresql --replicas=0
for i in $(seq 1 60); do $K -n zammad get pod zammad-postgresql-0 --ignore-not-found | grep -q zammad-postgresql-0 || break; sleep 2; done
cat <<'"'\\''EOF'\\'' | "'$K apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: zammad-recopy-pg
  namespace: zammad
spec:
  restartPolicy: Never
  nodeSelector:
    kubernetes.io/hostname: mbrkanov-mini-it13
  containers:
    - name: copy
      image: alpine:3.18.4
      imagePullPolicy: IfNotPresent
      command:
        - sh
        - -c
        - |
          set -eu
          rm -rf /new/* /new/.[!.]* /new/..?* 2>/dev/null || true
          cp -a /old/. /new/
          echo old_pg_version=$(cat /old/data/PG_VERSION)
          echo new_pg_version=$(cat /new/data/PG_VERSION)
          ls -l /old/data/PG_VERSION /new/data/PG_VERSION
          du -sh /old /new || true
      volumeMounts:
        - name: old
          mountPath: /old
          readOnly: true
        - name: new
          mountPath: /new
  volumes:
    - name: old
      persistentVolumeClaim:
        claimName: data-zammad-postgresql-0
    - name: new
      persistentVolumeClaim:
        claimName: zammad-postgresql-data-longhorn
EOF
for i in $(seq 1 600); do phase=$($K -n zammad get pod zammad-recopy-pg -o jsonpath='"'{.status.phase}' 2>/dev/null || true); echo phase="'$phase; [ "$phase" = Succeeded ] && break; [ "$phase" = Failed ] && break; sleep 3; done
$K -n zammad logs zammad-recopy-pg
$K -n zammad delete pod zammad-recopy-pg --wait=true
'"'"


Output:
statefulset.apps/zammad-postgresql scaled
pod/zammad-recopy-pg created
phase=Pending
phase=Pending
phase=Pending
phase=Pending
phase=Pending
phase=Succeeded
old_pg_version=15
new_pg_version=15
-rw-------    1 1001     root             3 Dec 31  2023 /new/data/PG_VERSION
-rw-------    1 1001     root             3 Dec 31  2023 /old/data/PG_VERSION
467.2M	/old
466.8M	/new
pod "zammad-recopy-pg" deleted


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'set -euo pipefail
K=\"sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml\"
"'$K -n zammad scale sts zammad-postgresql --replicas=1
$K -n zammad get sts,pods -o wide
'"'"


Output:
statefulset.apps/zammad-postgresql scaled
NAME                                           READY   AGE     CONTAINERS                                                          IMAGES
statefulset.apps/zammad                        0/1     3m36s   zammad-nginx,zammad-railsserver,zammad-scheduler,zammad-websocket   ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1
statefulset.apps/zammad-redis-master           1/1     5m8s    redis                                                               docker.io/bitnami/redis:7.0.11-debian-11-r20
statefulset.apps/zammad-elasticsearch-master   1/1     5m8s    elasticsearch                                                       docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     5m8s    postgresql                                                          docker.io/bitnami/postgresql:15.3.0-debian-11-r17

NAME                                    READY   STATUS              RESTARTS       AGE     IP           NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-memcached-68ff6f5577-nrs4k   1/1     Running             0              3m5s    10.42.0.80   mbrkanov-mini-it13   <none>           <none>
pod/zammad-redis-master-0               1/1     Running             0              3m4s    10.42.0.83   mbrkanov-mini-it13   <none>           <none>
pod/zammad-elasticsearch-master-0       1/1     Running             0              3m2s    10.42.0.81   mbrkanov-mini-it13   <none>           <none>
pod/zammad-0                            0/4     Init:Error          5 (102s ago)   3m36s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                 0/1     ContainerCreating   0              0s      <none>       mbrkanov-mini-it13   <none>           <none>


/bin/bash -lc 'for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
  echo "--- app recover $i"
  ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 '"'sudo kubectl --kubeconfig=<redacted> -n zammad get sts,pods -o wide'
  sleep 10
done"


/bin/bash -lc "ps -eo pid,ppid,stat,cmd | rg 'app recover|zammad get sts,pods'"


Output:
5600      33 Ss   /bin/bash -c for i in 1 2 3 4 5 6 7 8 9 10 11 12; do   echo "--- app recover $i"   ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad get sts,pods -o wide'   sleep 10 done
   5633      33 Ss   /bin/bash -c ps -eo pid,ppid,stat,cmd | rg 'app recover|zammad get sts,pods'
   5635    5633 S    rg app recover|zammad get sts,pods


Output:
NAME                                           READY   AGE     CONTAINERS                                                          IMAGES
statefulset.apps/zammad                        0/1     3m45s   zammad-nginx,zammad-railsserver,zammad-scheduler,zammad-websocket   ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1
statefulset.apps/zammad-redis-master           1/1     5m17s   redis                                                               docker.io/bitnami/redis:7.0.11-debian-11-r20
statefulset.apps/zammad-elasticsearch-master   1/1     5m17s   elasticsearch                                                       docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     5m17s   postgresql                                                          docker.io/bitnami/postgresql:15.3.0-debian-11-r17

NAME                                    READY   STATUS                  RESTARTS      AGE     IP           NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             3m14s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
pod/zammad-redis-master-0               1/1     Running                 0             3m13s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
pod/zammad-elasticsearch-master-0       1/1     Running                 0             3m11s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                 0/1     ContainerCreating       0             9s      <none>       mbrkanov-mini-it13   <none>           <none>
pod/zammad-0                            0/4     Init:CrashLoopBackOff   5 (20s ago)   3m45s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
--- app recover 2
NAME                                           READY   AGE     CONTAINERS                                                          IMAGES
statefulset.apps/zammad                        0/1     3m57s   zammad-nginx,zammad-railsserver,zammad-scheduler,zammad-websocket   ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1
statefulset.apps/zammad-redis-master           1/1     5m29s   redis                                                               docker.io/bitnami/redis:7.0.11-debian-11-r20
statefulset.apps/zammad-elasticsearch-master   1/1     5m29s   elasticsearch                                                       docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     5m29s   postgresql                                                          docker.io/bitnami/postgresql:15.3.0-debian-11-r17

NAME                                    READY   STATUS                  RESTARTS      AGE     IP           NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             3m26s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
pod/zammad-redis-master-0               1/1     Running                 0             3m25s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
pod/zammad-elasticsearch-master-0       1/1     Running                 0             3m23s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                 0/1     ContainerCreating       0             21s     <none>       mbrkanov-mini-it13   <none>           <none>
pod/zammad-0                            0/4     Init:CrashLoopBackOff   5 (32s ago)   3m57s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>
--- app recover 3
NAME                                           READY   AGE     CONTAINERS                                                          IMAGES
statefulset.apps/zammad                        0/1     4m8s    zammad-nginx,zammad-railsserver,zammad-scheduler,zammad-websocket   ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1
statefulset.apps/zammad-redis-master           1/1     5m40s   redis                                                               docker.io/bitnami/redis:7.0.11-debian-11-r20
statefulset.apps/zammad-elasticsearch-master   1/1     5m40s   elasticsearch                                                       docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     5m40s   postgresql                                                          docker.io/bitnami/postgresql:15.3.0-debian-11-r17

NAME                                    READY   STATUS                  RESTARTS      AGE     IP           NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             3m37s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
pod/zammad-redis-master-0               1/1     Running                 0             3m36s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
pod/zammad-elasticsearch-master-0       1/1     Running                 0             3m34s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                 0/1     ContainerCreating       0             32s     <none>       mbrkanov-mini-it13   <none>           <none>
pod/zammad-0                            0/4     Init:CrashLoopBackOff   5 (43s ago)   4m8s    10.42.0.79   mbrkanov-mini-it13   <none>           <none>
--- app recover 4
NAME                                           READY   AGE     CONTAINERS                                                          IMAGES
statefulset.apps/zammad                        0/1     4m19s   zammad-nginx,zammad-railsserver,zammad-scheduler,zammad-websocket   ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1,ghcr.io/zammad/zammad:6.2.0-1
statefulset.apps/zammad-redis-master           1/1     5m51s   redis                                                               docker.io/bitnami/redis:7.0.11-debian-11-r20
statefulset.apps/zammad-elasticsearch-master   1/1     5m51s   elasticsearch                                                       docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0
statefulset.apps/zammad-postgresql             0/1     5m51s   postgresql                                                          docker.io/bitnami/postgresql:15.3.0-debian-11-r17

NAME                                    READY   STATUS                  RESTARTS      AGE     IP           NODE                 NOMINATED NODE   READINESS GATES
pod/zammad-memcached-68ff6f5577-nrs4k   1/1     Running                 0             3m48s   10.42.0.80   mbrkanov-mini-it13   <none>           <none>
pod/zammad-redis-master-0               1/1     Running                 0             3m47s   10.42.0.83   mbrkanov-mini-it13   <none>           <none>
pod/zammad-elasticsearch-master-0       1/1     Running                 0             3m45s   10.42.0.81   mbrkanov-mini-it13   <none>           <none>
pod/zammad-postgresql-0                 0/1     ContainerCreating       0             43s     <none>       mbrkanov-mini-it13   <none>           <none>
pod/zammad-0                            0/4     Init:CrashLoopBackOff   5 (54s ago)   4m19s   10.42.0.79   mbrkanov-mini-it13   <none>           <none>


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad get pvc zammad-postgresql-data-longhorn -o yaml | sed -n \"1,160p\"'"


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n longhorn-system get volumes.longhorn.io 2>/dev/null | egrep \"NAME|pvc-e8dd|zammad\" || true'"


Output:
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"primary","app.kubernetes.io/instance":"zammad","app.kubernetes.io/name":"postgresql"},"name":"zammad-postgresql-data-longhorn","namespace":"zammad"},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"8Gi"}},"storageClassName":"longhorn"}}
    pv.kubernetes.io/bind-completed: "yes"
    pv.kubernetes.io/bound-by-controller: "yes"
    volume.beta.kubernetes.io/storage-provisioner: driver.longhorn.io
    volume.kubernetes.io/storage-provisioner: driver.longhorn.io
  creationTimestamp: "2026-06-17T22:18:20Z"
  finalizers:
  - kubernetes.io/pvc-protection
  labels:
    app.kubernetes.io/component: primary
    app.kubernetes.io/instance: zammad
    app.kubernetes.io/name: postgresql
  name: zammad-postgresql-data-longhorn
  namespace: zammad
  resourceVersion: "87494060"
  uid: e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 8Gi
  storageClassName: longhorn
  volumeMode: Filesystem
  volumeName: pvc-e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f
status:
  accessModes:
  - ReadWriteOnce
  capacity:
    storage: 8Gi
  phase: Bound


Output:
app.kubernetes.io/instance=zammad
                  app.kubernetes.io/managed-by=Helm
                  app.kubernetes.io/name=postgresql
                  apps.kubernetes.io/pod-index=0
                  controller-revision-hash=<redacted>
                  helm.sh/chart=<redacted>
                  statefulset.kubernetes.io/pod-name=<redacted>
Annotations:      <none>
Status:           Pending
IP:
IPs:              <none>
Controlled By:    StatefulSet/zammad-postgresql
Containers:
  postgresql:
    Container ID:
    Image:          docker.io/bitnami/postgresql:15.3.0-debian-11-r17
    Image ID:
    Port:           5432/TCP
    Host Port:      0/TCP
    State:          Waiting
      Reason:       ContainerCreating
    Ready:          False
    Restart Count:  0
    Requests:
      cpu:      250m
      memory:   256Mi
    Liveness:   exec [/bin/sh -c exec pg_isready -U "zammad" -d "dbname=zammad_production" -h 127.0.0.1 -p 5432] delay=30s timeout=5s period=10s #success=1 #failure=6
    Readiness:  exec [/bin/sh -c -e exec pg_isready -U "zammad" -d "dbname=zammad_production" -h 127.0.0.1 -p 5432
[ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ]
] delay=5s timeout=5s period=10s #success=1 #failure=6
    Environment:
      BITNAMI_DEBUG:                        false
      POSTGRESQL_PORT_NUMBER:               5432
      POSTGRESQL_VOLUME_DIR:                /bitnami/postgresql
      PGDATA:                               /bitnami/postgresql/data
      POSTGRES_USER:                        zammad
      POSTGRES_PASSWORD:                    <set to the key 'password' in secret 'zammad-postgresql'>  Optional: false
      POSTGRES_POSTGRES_PASSWORD:           zammad
      POSTGRES_DATABASE:                    zammad_production
      POSTGRESQL_ENABLE_LDAP:               no
      POSTGRESQL_ENABLE_TLS:                no
      POSTGRESQL_LOG_HOSTNAME:              false
      POSTGRESQL_LOG_CONNECTIONS:           false
      POSTGRESQL_LOG_DISCONNECTIONS:        false
      POSTGRESQL_PGAUDIT_LOG_CATALOG:       off
      POSTGRESQL_CLIENT_MIN_MESSAGES:       error
      POSTGRESQL_SHARED_PRELOAD_LIBRARIES:  pgaudit
    Mounts:
      /bitnami/postgresql from data (rw)
      /dev/shm from dshm (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-fsq94 (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  dshm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  zammad-postgresql-data-longhorn
    ReadOnly:   false
  kube-api-access-fsq94:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/hostname=<redacted>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type    Reason     Age   From               Message
  ----    ------     ----  ----               -------
  Normal  Scheduled  65s   default-scheduler  Successfully assigned zammad/zammad-postgresql-0 to mbrkanov-mini-it13


Output:
NAME                                       DATA ENGINE   STATE      ROBUSTNESS   SCHEDULED   SIZE           NODE                 AGE
pvc-e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f   v1            attached   healthy                  8589934592     mbrkanov-mini-it13   15m


/bin/bash -lc 'for i in 1 2 3 4 5 6; do
  echo "--- pg wait $i"
  ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 '"'sudo kubectl --kubeconfig=<redacted> -n zammad get pod zammad-postgresql-0 -o wide; sudo kubectl --kubeconfig=<redacted> -n zammad logs zammad-postgresql-0 --tail=30 2>/dev/null || true'
  sleep 10
done"


Output:
NAME                  READY   STATUS              RESTARTS   AGE   IP       NODE                 NOMINATED NODE   READINESS GATES
zammad-postgresql-0   0/1     ContainerCreating   0          77s   <none>   mbrkanov-mini-it13   <none>           <none>
--- pg wait 2
NAME                  READY   STATUS              RESTARTS   AGE   IP       NODE                 NOMINATED NODE   READINESS GATES
zammad-postgresql-0   0/1     ContainerCreating   0          88s   <none>   mbrkanov-mini-it13   <none>           <none>
--- pg wait 3
NAME                  READY   STATUS              RESTARTS   AGE   IP       NODE                 NOMINATED NODE   READINESS GATES
zammad-postgresql-0   0/1     ContainerCreating   0          99s   <none>   mbrkanov-mini-it13   <none>           <none>
--- pg wait 4
NAME                  READY   STATUS              RESTARTS   AGE    IP       NODE                 NOMINATED NODE   READINESS GATES
zammad-postgresql-0   0/1     ContainerCreating   0          110s   <none>   mbrkanov-mini-it13   <none>           <none>
--- pg wait 5
NAME                  READY   STATUS              RESTARTS   AGE    IP       NODE                 NOMINATED NODE   READINESS GATES
zammad-postgresql-0   0/1     ContainerCreating   0          2m1s   <none>   mbrkanov-mini-it13   <none>           <none>
--- pg wait 6
NAME                  READY   STATUS              RESTARTS   AGE     IP       NODE                 NOMINATED NODE   READINESS GATES
zammad-postgresql-0   0/1     ContainerCreating   0          2m12s   <none>   mbrkanov-mini-it13   <none>           <none>


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad get events --sort-by=.lastTimestamp | tail -n 40'"


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo kubectl --kubeconfig=<redacted> -n zammad describe pod zammad-postgresql-0 | sed -n \"/Events:/,"'$p"'"'"


/bin/bash -lc "ssh -o BatchMode=yes -o ConnectTimeout=8 -i /home/node/.ssh/kube-bot-it15-ed25519 kube-bot@192.168.20.40 'sudo journalctl -u k3s --since \"5 minutes ago\" --no-pager | egrep -i \"zammad-postgresql|pvc-e8dd|MountVolume|longhorn|error|failed\" | tail -n 80'"


Output:
5m39s       Warning   RecreatingFailedPod      statefulset/zammad-elasticsearch-master                    StatefulSet zammad/zammad-elasticsearch-master is recreating failed Pod zammad-elasticsearch-master-0
5m39s       Normal    SuccessfulDelete         statefulset/zammad-elasticsearch-master                    delete Pod zammad-elasticsearch-master-0 in StatefulSet zammad-elasticsearch-master successful
5m37s       Normal    SuccessfulDelete         replicaset/zammad-memcached-95c5bfd6b                      Deleted pod: zammad-memcached-95c5bfd6b-h6m6v
5m37s       Normal    ScalingReplicaSet        deployment/zammad-memcached                                Scaled down replica set zammad-memcached-95c5bfd6b to 0 from 1
5m28s       Normal    Pulled                   pod/zammad-elasticsearch-master-0                          Container image "docker.io/bitnami/bitnami-shell:11-debian-11-r131" already present on machine
5m28s       Normal    Created                  pod/zammad-elasticsearch-master-0                          Created container sysctl
5m28s       Normal    Started                  pod/zammad-elasticsearch-master-0                          Started container sysctl
5m27s       Normal    Created                  pod/zammad-elasticsearch-master-0                          Created container elasticsearch
5m27s       Normal    SuccessfulAttachVolume   pod/zammad-redis-master-0                                  AttachVolume.Attach succeeded for volume "pvc-61d812f1-d3f3-4d63-a194-782128ae536f"
5m27s       Normal    Pulled                   pod/zammad-elasticsearch-master-0                          Container image "docker.io/bitnami/elasticsearch:8.8.2-debian-11-r0" already present on machine
5m27s       Normal    Started                  pod/zammad-elasticsearch-master-0                          Started container elasticsearch
5m27s       Normal    SuccessfulAttachVolume   pod/zammad-postgresql-0                                    AttachVolume.Attach succeeded for volume "pvc-e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f"
5m23s       Normal    Created                  pod/zammad-redis-master-0                                  Created container redis
5m23s       Normal    Started                  pod/zammad-redis-master-0                                  Started container redis
5m23s       Normal    Pulled                   pod/zammad-redis-master-0                                  Container image "docker.io/bitnami/redis:7.0.11-debian-11-r20" already present on machine
4m39s       Normal    Created                  pod/zammad-postgresql-0                                    Created container postgresql
4m39s       Normal    Started                  pod/zammad-postgresql-0                                    Started container postgresql
4m39s       Normal    Pulled                   pod/zammad-postgresql-0                                    Container image "docker.io/bitnami/postgresql:15.3.0-debian-11-r17" already present on machine
4m34s       Warning   BackOff                  pod/zammad-postgresql-0                                    Back-off restarting failed container postgresql in pod zammad-postgresql-0_zammad(1f729811-bb38-4347-b270-a76f6979da79)
4m28s       Normal    SuccessfulDelete         statefulset/zammad-postgresql                              delete Pod zammad-postgresql-0 in StatefulSet zammad-postgresql successful
4m21s       Normal    Pulled                   pod/zammad-0                                               Container image "ghcr.io/zammad/zammad:6.2.0-1" already present on machine
4m21s       Normal    Created                  pod/zammad-0                                               Created container postgresql-init
4m21s       Normal    Started                  pod/zammad-0                                               Started container postgresql-init
4m18s       Normal    Pulled                   pod/zammad-inspect-pg-pvcs                                 Container image "alpine:3.18.4" already present on machine
4m18s       Normal    Created                  pod/zammad-inspect-pg-pvcs                                 Created container inspect
4m18s       Normal    Started                  pod/zammad-inspect-pg-pvcs                                 Started container inspect
3m29s       Normal    SuccessfulAttachVolume   pod/zammad-inspect-pg-pvcs                                 AttachVolume.Attach succeeded for volume "pvc-e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f"
3m26s       Normal    Pulled                   pod/zammad-inspect-pg-pvcs                                 Container image "alpine:3.18.4" already present on machine
3m26s       Normal    Created                  pod/zammad-inspect-pg-pvcs                                 Created container inspect
3m26s       Normal    Started                  pod/zammad-inspect-pg-pvcs                                 Started container inspect
2m52s       Normal    SuccessfulAttachVolume   pod/zammad-recopy-pg                                       AttachVolume.Attach succeeded for volume "pvc-e8dd7160-96c9-4409-b9b8-0e15e1a2ba5f"
2m50s       Normal    Pulled                   pod/zammad-recopy-pg                                       Container image "alpine:3.18.4" already present on machine
2m50s       Normal    Created                  pod/zammad-recopy-pg                                       Created container copy
2m50s       Normal    Started                  pod/zammad-recopy-pg                                       Started container copy
2m37s       Normal    SuccessfulCreate         statefulset/zammad-postgresql                              create Pod zammad-postgresql-0 in StatefulSet zammad-postgresql successful
2m32s       Normal    Valid                    secretstore/dmz-infisical                                  store validated
50s         Warning   BackOff                  pod/zammad-0                                               Back-off restarting failed container postgresql-init in pod zammad-0_zammad(08c3a777-d821-4b25-9ed1-6a82167895e1)
21s         Normal    Pulled                   pod/zammad-postgresql-0                                    Container image "docker.io/bitnami/postgresql:15.3.0-debian-11-r17" already present on machine
21s         Normal    Created                  pod/zammad-postgresql-0                                    Created container postgresql
21s         Normal    Started                  pod/zammad-postgresql-0                                    Started container postgresql