Skip to content

Concurrent map write regression #40097

@howardjohn

Description

@howardjohn

Tests are failing with concurrent map writes after #40036: https://prow.istio.io/view/gs/istio-prow/logs/integ-pilot-multicluster_istio_postsubmit/1551562663899697152, https://storage.googleapis.com/istio-prow/logs/integ-pilot-multicluster_istio_postsubmit/1551562663899697152/artifacts/pilot-a925280f3e3e42c39c5946a2e/TestCustomGateway/helm/_test_context/istio-state4186193048/primary/istiod-7f86f4b6c4-6th8b_discovery.previous.log

fatal error: concurrent map read and map write

goroutine 179 [running]:
runtime.throw({0x33b4e67?, 0x2b3d540?})
	runtime/panic.go:992 +0x71 fp=0xc0045d92e8 sp=0xc0045d92b8 pc=0x438651
runtime.mapaccess1_faststr(0x2d160e0?, 0xc007abafc0?, {0x337d45a, 0xe})
	runtime/map_faststr.go:22 +0x3a5 fp=0xc0045d9350 sp=0xc0045d92e8 pc=0x412e45
istio.io/istio/pilot/pkg/serviceregistry/kube/controller.(*Controller).getPodLocality(0xc00035c780, 0xc0047f0800)
	istio.io/istio/pilot/pkg/serviceregistry/kube/controller/controller.go:882 +0x5b fp=0xc0045d9748 sp=0xc0045d9350 pc=0x27a30fb
istio.io/istio/pilot/pkg/serviceregistry/kube/controller.NewEndpointBuilder({0x383eb60?, 0xc00035c780}, 0xc0047f0800)
	istio.io/istio/pilot/pkg/serviceregistry/kube/controller/endpoint_builder.go:53 +0x9a fp=0xc0045d9af0 sp=0xc0045d9748 pc=0x27a97da
istio.io/istio/pilot/pkg/serviceregistry/kube/controller.(*PodCache).notifyWorkloadHandlers(0xc0000d8230, 0xc0047f0800, 0xa?)
	istio.io/istio/pilot/pkg/serviceregistry/kube/controller/pod.go:189 +0x7c fp=0xc0045d9d10 sp=0xc0045d9af0 pc=0x27b949c
istio.io/istio/pilot/pkg/serviceregistry/kube/controller.(*PodCache).onEvent(0xc00008c500?, {0x3334800?, 0xc0047f0800?}, 0x1)
	istio.io/istio/pilot/pkg/serviceregistry/kube/controller/pod.go:178 +0x406 fp=0xc0045d9d98 sp=0xc0045d9d10 pc=0x27b9346
istio.io/istio/pilot/pkg/serviceregistry/kube/controller.(*PodCache).onEvent-fm({0x3334800?, 0xc0047f0800?}, 0x3334800?)
	<autogenerated>:1 +0x3c fp=0xc0045d9dc8 sp=0xc0045d9d98 pc=0x27bf0dc
istio.io/istio/pilot/pkg/serviceregistry/kube/controller.(*Controller).registerHandlers.func1({0x3334800?, 0xc0047f0800?}, 0xc0017b2650?)
	istio.io/istio/pilot/pkg/serviceregistry/kube/controller/controller.go:661 +0x57 fp=0xc0045d9e08 sp=0xc0045d9dc8 pc=0x27a1ad7
istio.io/istio/pilot/pkg/serviceregistry/kube/controller.(*Controller).registerHandlers.func3.1()
	istio.io/istio/pilot/pkg/serviceregistry/kube/controller/controller.go:690 +0x2a fp=0xc0045d9e30 sp=0xc0045d9e08 pc=0x27a18aa
istio.io/istio/pkg/queue.(*queueImpl).Run(0xc0007d4ba0, 0xc0006507e0)
	istio.io/istio/pkg/queue/instance.go:157 +0x212 fp=0xc0045d9ef0 sp=0xc0045d9e30 pc=0x1b1be52
istio.io/istio/pilot/pkg/serviceregistry/kube/controller.(*Controller).Run(0xc00035c780, 0x0?)
	istio.io/istio/pilot/pkg/serviceregistry/kube/controller/controller.go:848 +0x2da fp=0xc0045d9fc0 sp=0xc0045d9ef0 pc=0x27a2b1a
istio.io/istio/pilot/pkg/serviceregistry/aggregate.(*Controller).Run.func1()
	istio.io/istio/pilot/pkg/serviceregistry/aggregate/controller.go:328 +0x2f fp=0xc0045d9fe0 sp=0xc0045d9fc0 pc=0x26c80cf
runtime.goexit()
	runtime/asm_amd64.s:1571 +0x1 fp=0xc0045d9fe8 sp=0xc0045d9fe0 pc=0x46b7e1
created by istio.io/istio/pilot/pkg/serviceregistry/aggregate.(*Controller).Run
	istio.io/istio/pilot/pkg/serviceregistry/aggregate/controller.go:328 +0x58

Metadata

Metadata

Assignees

Labels

No labels
No labels

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions