libp2p slow consumer events #10778
Description
Checklist
- This is a bug report, not a question. Ask questions on discuss.ipfs.tech.
- I have searched on the issue tracker for my bug.
- I am running the latest kubo version or have an issue updating.
Installation method
docker image
Version
Kubo version: 0.34.1-4649554
Repo version: 16
System version: amd64/linux
Golang version: go1.24.1
Config
{
"API": {
"HTTPHeaders": {}
},
"Addresses": {
"API": "/ip4/0.0.0.0/tcp/5001",
"Announce": <removed>
"AppendAnnounce": [],
"Gateway": "/ip4/0.0.0.0/tcp/8080",
"NoAnnounce": [
"/ip4/10.0.0.0/ipcidr/8",
"/ip4/100.64.0.0/ipcidr/10",
"/ip4/169.254.0.0/ipcidr/16",
"/ip4/172.16.0.0/ipcidr/12",
"/ip4/192.0.0.0/ipcidr/24",
"/ip4/192.0.2.0/ipcidr/24",
"/ip4/192.168.0.0/ipcidr/16",
"/ip4/198.18.0.0/ipcidr/15",
"/ip4/198.51.100.0/ipcidr/24",
"/ip4/203.0.113.0/ipcidr/24",
"/ip4/240.0.0.0/ipcidr/4",
"/ip6/100::/ipcidr/64",
"/ip6/2001:2::/ipcidr/48",
"/ip6/2001:db8::/ipcidr/32",
"/ip6/fc00::/ipcidr/7",
"/ip6/fe80::/ipcidr/10"
],
"Swarm": [
"/ip4/0.0.0.0/tcp/4001",
"/ip6/::/tcp/4001",
"/ip4/0.0.0.0/udp/4001/webrtc-direct",
"/ip4/0.0.0.0/udp/4001/quic-v1",
"/ip4/0.0.0.0/udp/4001/quic-v1/webtransport",
"/ip6/::/udp/4001/webrtc-direct",
"/ip6/::/udp/4001/quic-v1",
"/ip6/::/udp/4001/quic-v1/webtransport"
]
},
"AutoNAT": {},
"AutoTLS": {},
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"
],
"DNS": {
"Resolvers": {}
},
"Datastore": {
"BlockKeyCacheSize": null,
"BloomFilterSize": 0,
"GCPeriod": "1h",
"HashOnRead": false,
"Spec": {
"mounts": [
{
"child": {
"path": "blocks",
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/3",
"sync": true,
"type": "flatfs"
},
"mountpoint": "/blocks",
"prefix": "flatfs.datastore",
"type": "measure"
},
{
"child": {
"disableWAL": false,
"path": "datastore",
"type": "pebbleds"
},
"mountpoint": "/",
"prefix": "pebble.datastore",
"type": "measure"
}
],
"type": "mount"
},
"StorageGCWatermark": 90,
"StorageMax": "1TB"
},
"Discovery": {
"MDNS": {
"Enabled": false
}
},
"Experimental": {
"FilestoreEnabled": false,
"Libp2pStreamMounting": false,
"OptimisticProvide": false,
"OptimisticProvideJobsPoolSize": 0,
"P2pHttpProxy": false,
"StrategicProviding": false,
"UrlstoreEnabled": false
},
"Gateway": {
"DeserializedResponses": null,
"DisableHTMLErrors": null,
"ExposeRoutingAPI": null,
"HTTPHeaders": {},
"NoDNSLink": true,
"NoFetch": true,
"PublicGateways": {
"localhost": {
"Paths": [
"/ipfs"
],
"UseSubdomains": false
}
},
"RootRedirect": ""
},
"Identity": {
"PeerID": <removed>
},
"Import": {
"BatchMaxNodes": null,
"BatchMaxSize": null,
"CidVersion": null,
"HashFunction": null,
"UnixFSChunker": null,
"UnixFSRawLeaves": null
},
"Internal": {},
"Ipns": {
"RecordLifetime": "",
"RepublishPeriod": "",
"ResolveCacheSize": 128
},
"Migration": {
"DownloadSources": [],
"Keep": ""
},
"Mounts": {
"FuseAllowOther": false,
"IPFS": "/ipfs",
"IPNS": "/ipns"
},
"Peering": {
"Peers": <removed>
},
"Pinning": {
"RemoteServices": {}
},
"Plugins": {
"Plugins": null
},
"Provider": {
"Strategy": ""
},
"Pubsub": {
"DisableSigning": false,
"Router": ""
},
"Reprovider": {},
"Routing": {
"AcceleratedDHTClient": true,
"Methods": null,
"Routers": null
},
"Swarm": {
"AddrFilters": [
"/ip4/10.0.0.0/ipcidr/8",
"/ip4/100.64.0.0/ipcidr/10",
"/ip4/169.254.0.0/ipcidr/16",
"/ip4/172.16.0.0/ipcidr/12",
"/ip4/192.0.0.0/ipcidr/24",
"/ip4/192.0.2.0/ipcidr/24",
"/ip4/192.168.0.0/ipcidr/16",
"/ip4/198.18.0.0/ipcidr/15",
"/ip4/198.51.100.0/ipcidr/24",
"/ip4/203.0.113.0/ipcidr/24",
"/ip4/240.0.0.0/ipcidr/4",
"/ip6/100::/ipcidr/64",
"/ip6/2001:2::/ipcidr/48",
"/ip6/2001:db8::/ipcidr/32",
"/ip6/fc00::/ipcidr/7",
"/ip6/fe80::/ipcidr/10"
],
"ConnMgr": {
"GracePeriod": "30s",
"HighWater": 2048,
"LowWater": 1024
},
"DisableBandwidthMetrics": false,
"DisableNatPortMap": true,
"RelayClient": {
"Enabled": false
},
"RelayService": {
"Enabled": false
},
"ResourceMgr": {},
"Transports": {
"Multiplexers": {},
"Network": {},
"Security": {}
}
},
"Version": {}
}Description
Hello,
We are currently testing Kubo version 0.34.1 since we are trying to migrate from a much older version. We are currently in the process of testing re-pinning everything (for our use case we have to pin via the network instead of adding) in the new cluster which seems to work fine, but we noticed some new errors in the logs relating to libp2p:
2025-04-09T01:53:07.915Z ERROR eventbus eventbus/basic.go:467 subscriber named "autonat" is a slow consumer of event.EvtPeerIdentificationCompleted. This can lead to libp2p stalling and hard to debug issues.
2025-04-09T01:53:08.976Z ERROR eventbus eventbus/basic.go:467 subscriber named "libp2p/go-libp2p-kad-dht/subscriber_notifee.go-L37" is a slow consumer of event.EvtPeerConnectednessChanged. This can lead to libp2p stalling and hard to debug issues.
2025-04-09T01:53:09.017Z ERROR eventbus eventbus/basic.go:467 subscriber named "libp2p/go-libp2p-kad-dht/subscriber_notifee.go-L37" is a slow consumer of event.EvtPeerProtocolsUpdated. This can lead to libp2p stalling and hard to debug issues.
2025-04-09T07:32:23.017Z ERROR eventbus eventbus/basic.go:467 subscriber named "pstoremanager" is a slow consumer of event.EvtPeerConnectednessChanged. This can lead to libp2p stalling and hard to debug issues.
These errors seem to occur when there is a lot of pinning activity on the nodes with CPU and memory metrics going up, although not something we wouldn't expect in this case.
These errors do not seem have a visible impact on the operations, although we I couldn't really test the libp2p part in case if these is stalling as mentioned in the error logs.
One thing to note is that we are testing the use of AWS EFS as the backing storage medium for Kubo, but not sure if that's related (all metrics look nominal). Another thing to note is that we are using ipfs-cluster on top of Kubo to do the pinning and manage the nodes, again nothing sus there.
Please let me know if there is any more info we could gather to help investigate.
Thanks!