-
Notifications
You must be signed in to change notification settings - Fork 3.4k
Description
Is there an existing issue for this?
- I have searched the existing issues
What happened?
Conntrack (ct) entry remaining lifetime (cilium bpf ct list global -d
) is converted incorrectly, seemingly from #25795.
With the change, the remaining time if reported as a massive negative number (-4B), example:
TCP IN 169.254.42.1:55555 -> 10.244.0.6:80 expires=16817045 (remaining: -4391264792 sec(s)) RxPackets=6 RxBytes=489 RxFlagsSeen=0x1b LastRxReport=16817036 TxPackets=5 TxBytes=468 TxFlagsSeen=0x1b LastTxReport=16817036 Flags=0x001b [ RxClosing TxClosing LBLoopback SeenNonSyn ] RevNAT=2 SourceSecurityID=12079 IfIndex=0
Cilium Version
Client: 1.14.0-dev 8531c5a 2023-05-31T15:22:03+02:00 go version go1.20.4 linux/amd64
Daemon: 1.14.0-dev 8531c5a 2023-05-31T15:22:03+02:00 go version go1.20.4 linux/amd64
Kernel Version
Linux 6.1.25-1rodete1-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.25-1rodete1 (2023-05-11) x86_64 GNU/Linux
Kubernetes Version
N/A
Sysdump
N/A
Relevant log output
TCP IN 169.254.42.1:55555 -> 10.244.0.6:80 expires=16817045 (remaining: -4391264792 sec(s)) RxPackets=6 RxBytes=489 RxFlagsSeen=0x1b LastRxReport=16817036 TxPackets=5 TxBytes=468 TxFlagsSeen=0x1b LastTxReport=16817036 Flags=0x001b [ RxClosing TxClosing LBLoopback SeenNonSyn ] RevNAT=2 SourceSecurityID=12079 IfIndex=0
Anything else?
Reproducible using commit 8531c5a7
from #25795
git checkout 8531c5a7
KUBEPROXY_MODE="none" make kind && \
make kind-image && \
helm upgrade -i cilium ./install/kubernetes/cilium \
--wait \
--namespace kube-system \
--set k8sServiceHost="kind-control-plane" \
--set k8sServicePort="6443" \
--set debug.enabled=true \
--set pprof.enabled=true \
--set enableIPv4Masquerade=false \
--set enableIPv6Masquerade=false \
--set enableIPv4Masquerade=false \
--set hostFirewall.enabled=false \
--set socketLB.hostNamespaceOnly=true \
--set kubeProxyReplacement=strict \
--set nodeinit.enabled=true \
--set ipam.mode=kubernetes \
--set ipv4.enabled=true \
--set ipv4NativeRoutingCIDR=10.244.0.0/16 \
--set ipv6.enabled=false \
--set image.override="localhost:5000/cilium/cilium-dev:local" \
--set image.pullPolicy=Never \
--set operator.image.override="localhost:5000/cilium/operator-generic:local" \
--set operator.image.pullPolicy=Never \
--set operator.image.suffix="" \
--set securityContext.privileged=true \
--set gatewayAPI.enabled=false \
--set socketLB.enabled=false \
--set loadbalancer.serviceTopology=false \
--set localRedirectPolicy=true \
--set bpf.hostLegacyRouting=true \
--set enableCiliumEndpointSlice=true \
--set endpointRoutes.enabled=true
alias kk="kubectl --context kind-kind"
kk --context kind-kind apply -f - <<EOF
apiVersion: v1
kind: Pod
metadata:
name: server
namespace: default
labels:
app: server
spec:
containers:
- name: agnhost
image: k8s.gcr.io/e2e-test-images/agnhost:2.39
args:
- netexec
- --http-port=80
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: server
spec:
type: ClusterIP
selector:
app: server
ports:
- protocol: TCP
port: 80
targetPort: 80
EOF
time while ! [[ "$( kk get svc server -ojsonpath='{.spec.clusterIP}' | xargs )" =~ ^10\..* ]]; do echo -n "."; sleep 1; done
clusterip=$( kk get svc server -ojsonpath='{.spec.clusterIP}' )
time while ! [[ "$( kk get pod server -ojsonpath='{.status.conditions[?(@.type=="Ready")].status}' )" =~ "True" ]]; do echo -n "."; sleep 1; done
kk exec server -- curl --no-keepalive --connect-timeout 5 --local-port 55555 -v ${clusterip}:80/hostname
srchost=$( kk get pod server \
-ojsonpath='{ .spec.nodeName }' )
srcanetd=$( kk get pod -n kube-system \
-l k8s-app=cilium \
--field-selector=spec.nodeName=${srchost} \
-ojsonpath='{.items[].metadata.name}' )
kk exec -it -n kube-system ${srcanetd} -- cilium bpf ct list global -d | grep ":55555" | grep "169.254.42.1"
TCP IN 169.254.42.1:55555 -> 10.244.0.143:80 expires=16824145 (remaining: -4393118610 sec(s)) RxPackets=6 RxBytes=488 RxFlagsSeen=0x1b LastRxReport=16824136 TxPackets=5 TxBytes=468 TxFlagsSeen=0x1b LastTxReport=16824136 Flags=0x001b [ RxClosing TxClosing LBLoopback SeenNonSyn ] RevNAT=2 SourceSecurityID=13684 IfIndex=0
< TCP OUT 169.254.42.1:55555 -> 10.244.0.143:80 expires=16845229 (remaining: -4393097020 sec(s)) RxPackets=0 RxBytes=0 RxFlagsSeen=0x00 LastRxReport=0 TxPackets=6 TxBytes=488 TxFlagsSeen=0x1b LastTxReport=16824136 Flags=0x001a [ TxClosing LBLoopback SeenNonSyn ] RevNAT=2 SourceSecurityID=13684 IfIndex=0
Using the previous commit 0101ff43
, the issue is not reproduced:
git checkout 0101ff43
KUBEPROXY_MODE="none" make kind && \
make kind-image && \
helm upgrade -i cilium ./install/kubernetes/cilium \
--wait \
--namespace kube-system \
--set k8sServiceHost="kind-control-plane" \
--set k8sServicePort="6443" \
--set debug.enabled=true \
--set pprof.enabled=true \
--set enableIPv4Masquerade=false \
--set enableIPv6Masquerade=false \
--set enableIPv4Masquerade=false \
--set hostFirewall.enabled=false \
--set socketLB.hostNamespaceOnly=true \
--set kubeProxyReplacement=strict \
--set nodeinit.enabled=true \
--set ipam.mode=kubernetes \
--set ipv4.enabled=true \
--set ipv4NativeRoutingCIDR=10.244.0.0/16 \
--set ipv6.enabled=false \
--set image.override="localhost:5000/cilium/cilium-dev:local" \
--set image.pullPolicy=Never \
--set operator.image.override="localhost:5000/cilium/operator-generic:local" \
--set operator.image.pullPolicy=Never \
--set operator.image.suffix="" \
--set securityContext.privileged=true \
--set gatewayAPI.enabled=false \
--set socketLB.enabled=false \
--set loadbalancer.serviceTopology=false \
--set localRedirectPolicy=true \
--set bpf.hostLegacyRouting=true \
--set enableCiliumEndpointSlice=true \
--set endpointRoutes.enabled=true
alias kk="kubectl --context kind-kind"
kk --context kind-kind apply -f - <<EOF
apiVersion: v1
kind: Pod
metadata:
name: server
namespace: default
labels:
app: server
spec:
containers:
- name: agnhost
image: k8s.gcr.io/e2e-test-images/agnhost:2.39
args:
- netexec
- --http-port=80
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: server
spec:
type: ClusterIP
selector:
app: server
ports:
- protocol: TCP
port: 80
targetPort: 80
EOF
time while ! [[ "$( kk get svc server -ojsonpath='{.spec.clusterIP}' | xargs )" =~ ^10\..* ]]; do echo -n "."; sleep 1; done
clusterip=$( kk get svc server -ojsonpath='{.spec.clusterIP}' )
time while ! [[ "$( kk get pod server -ojsonpath='{.status.conditions[?(@.type=="Ready")].status}' )" =~ "True" ]]; do echo -n "."; sleep 1; done
kk exec server -- curl --no-keepalive --connect-timeout 5 --local-port 55555 -v ${clusterip}:80/hostname
srchost=$( kk get pod server \
-ojsonpath='{ .spec.nodeName }' )
srcanetd=$( kk get pod -n kube-system \
-l k8s-app=cilium \
--field-selector=spec.nodeName=${srchost} \
-ojsonpath='{.items[].metadata.name}' )
kk exec -it -n kube-system ${srcanetd} -- cilium bpf ct list global -d | grep ":55555" | grep "169.254.42.1"
TCP IN 169.254.42.1:55555 -> 10.244.1.80:80 expires=16823889 (remaining: 9 sec(s)) RxPackets=12 RxBytes=976 RxFlagsSeen=0x1b LastRxReport=16823880 TxPackets=9 TxBytes=862 TxFlagsSeen=0x1b LastTxReport=16823880 Flags=0x001b [ RxClosing TxClosing LBLoopback SeenNonSyn ] RevNAT=2 SourceSecurityID=21816 IfIndex=0
TCP OUT 169.254.42.1:55555 -> 10.244.1.80:80 expires=16844973 (remaining: 21599 sec(s)) RxPackets=0 RxBytes=0 RxFlagsSeen=0x00 LastRxReport=0 TxPackets=12 TxBytes=976 TxFlagsSeen=0x1b LastTxReport=16823880 Flags=0x001a [ TxClosing LBLoopback SeenNonSyn ] RevNAT=2 SourceSecurityID=21816 IfIndex=0
Code of Conduct
- I agree to follow this project's Code of Conduct