Skip to content

crash in s3-aws doWalk() since #2455 was merged #2553

@hrak

Description

@hrak

Since PR #2455 was merged, the registry is crashing with the following backtrace on the first hit it gets:

time="2018-03-20T08:23:52.237673588Z" level=info msg="debug server listening :5001"
time="2018-03-20T08:23:52.241709386Z" level=info msg="redis not configured" go.version=go1.10 instance.id=85be40ec-6e71-409a-bb3d-e67d3ceab702 service=registry version=v2.6.0-rc.1-273-g607ae5d1
time="2018-03-20T08:23:52.241842948Z" level=info msg="Starting upload purge in 11m0s" go.version=go1.10 instance.id=85be40ec-6e71-409a-bb3d-e67d3ceab702 service=registry version=v2.6.0-rc.1-273-g607ae5d1
time="2018-03-20T08:23:52.277934092Z" level=info msg="backend redirection disabled" go.version=go1.10 instance.id=85be40ec-6e71-409a-bb3d-e67d3ceab702 service=registry version=v2.6.0-rc.1-273-g607ae5d1
time="2018-03-20T08:23:52.277986941Z" level=info msg="using inmemory blob descriptor cache" go.version=go1.10 instance.id=85be40ec-6e71-409a-bb3d-e67d3ceab702 service=registry version=v2.6.0-rc.1-273-g607ae5d1
time="2018-03-20T08:23:52.278086413Z" level=info msg="listening on [::]:5000" go.version=go1.10 instance.id=85be40ec-6e71-409a-bb3d-e67d3ceab702 service=registry version=v2.6.0-rc.1-273-g607ae5d1
time="2018-03-20T08:31:53.603909559Z" level=panic msg="runtime error: invalid memory address or nil pointer dereference"
2018/03/20 08:31:53 http: panic serving 172.17.0.3:33060: &{0xc42016a0a0 map[] 2018-03-20 08:31:53.603909559 +0000 UTC m=+481.379379642 panic runtime error: invalid memory address or nil pointer dereference <nil>}
goroutine 31 [running]:
net/http.(*conn).serve.func1(0xc42036cbe0)
	/usr/local/opt/go/libexec/src/net/http/server.go:1726 +0xd0
panic(0xd678c0, 0xc420160780)
	/usr/local/opt/go/libexec/src/runtime/panic.go:505 +0x229
github.com/docker/distribution/vendor/github.com/sirupsen/logrus.Entry.log(0xc42016a0a0, 0xc4205e4090, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/vendor/github.com/sirupsen/logrus/entry.go:124 +0x568
github.com/docker/distribution/vendor/github.com/sirupsen/logrus.(*Entry).Panic(0xc420160730, 0xc420642960, 0x1, 0x1)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/vendor/github.com/sirupsen/logrus/entry.go:169 +0xaa
github.com/docker/distribution/vendor/github.com/sirupsen/logrus.(*Logger).Panic(0xc42016a0a0, 0xc420642960, 0x1, 0x1)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/vendor/github.com/sirupsen/logrus/logger.go:236 +0x6d
github.com/docker/distribution/vendor/github.com/sirupsen/logrus.Panic(0xc420642960, 0x1, 0x1)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/vendor/github.com/sirupsen/logrus/exported.go:107 +0x4b
github.com/docker/distribution/registry.panicHandler.func1.1()
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/registry.go:311 +0xf9
panic(0xc70780, 0x12f5e70)
	/usr/local/opt/go/libexec/src/runtime/panic.go:505 +0x229
github.com/docker/distribution/registry/storage/driver/s3-aws.(*driver).doWalk.func1(0xc420168c80, 0xc42022c001, 0x7fed1a013400)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/storage/driver/s3-aws/s3.go:946 +0xa0
github.com/docker/distribution/vendor/github.com/aws/aws-sdk-go/service/s3.(*S3).ListObjectsV2PagesWithContext(0xc4201660a0, 0x7fed1a013400, 0xc4204d7030, 0xc4205aec30, 0xc420642f38, 0x0, 0x0, 0x0, 0x1, 0x2)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/vendor/github.com/aws/aws-sdk-go/service/s3/api.go:4190 +0x10c
github.com/docker/distribution/registry/storage/driver/s3-aws.(*driver).doWalk(0xc420470d80, 0xe5c740, 0xc4204d6fc0, 0xc420642ff8, 0xc4200f8ae1, 0x20, 0xd7d645, 0x1, 0xc4205aebe0, 0x0, ...)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/storage/driver/s3-aws/s3.go:944 +0x3b6
github.com/docker/distribution/registry/storage/driver/s3-aws.(*driver).Walk(0xc420470d80, 0xe5c740, 0xc4204d6fc0, 0xc420105260, 0x20, 0xc4205aebe0, 0x2, 0x0)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/storage/driver/s3-aws/s3.go:892 +0x16b
github.com/docker/distribution/registry/storage/driver/base.(*Base).Walk(0xc4201647b0, 0xe5c740, 0xc4204d6fc0, 0xc420105260, 0x20, 0xc4205aebe0, 0x0, 0x0)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/storage/driver/base/base.go:239 +0x23d
github.com/docker/distribution/registry/storage.(*registry).Repositories(0xc4203886c0, 0xe5c8c0, 0xc4205ae5a0, 0xc4202f17c0, 0x14, 0x14, 0xc4200f8646, 0x0, 0xc4206431e8, 0x8204cf, ...)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/storage/catalog.go:29 +0x215
github.com/docker/distribution/registry/handlers.(*catalogHandler).GetCatalog(0xc420166390, 0xe5a540, 0xc4205ae410, 0xc42015b800)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/handlers/catalog.go:48 +0x12a
github.com/docker/distribution/registry/handlers.(*catalogHandler).GetCatalog-fm(0xe5a540, 0xc4205ae410, 0xc42015b800)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/handlers/catalog.go:24 +0x48
net/http.HandlerFunc.ServeHTTP(0xc4203abaa0, 0xe5a540, 0xc4205ae410, 0xc42015b800)
	/usr/local/opt/go/libexec/src/net/http/server.go:1947 +0x44
github.com/docker/distribution/vendor/github.com/gorilla/handlers.MethodHandler.ServeHTTP(0xc4202438c0, 0xe5a540, 0xc4205ae410, 0xc42015b800)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/vendor/github.com/gorilla/handlers/handlers.go:35 +0x34d
github.com/docker/distribution/registry/handlers.(*App).dispatcher.func1(0xe5a540, 0xc4205ae410, 0xc42015b800)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/handlers/app.go:715 +0x4a9
net/http.HandlerFunc.ServeHTTP(0xc42016c640, 0xe5a540, 0xc4205ae410, 0xc42015b200)
	/usr/local/opt/go/libexec/src/net/http/server.go:1947 +0x44
github.com/docker/distribution/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc4200aed20, 0xe5a540, 0xc4205ae410, 0xc42015b200)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/vendor/github.com/gorilla/mux/mux.go:114 +0xdc
github.com/docker/distribution/registry/handlers.(*App).ServeHTTP(0xc4200e6360, 0x7fed1a013210, 0xc4202429f0, 0xc42015b000)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/handlers/app.go:630 +0x2b4
github.com/docker/distribution/registry.alive.func1(0x7fed1a013210, 0xc4202429f0, 0xc42015af00)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/registry.go:331 +0x6a
net/http.HandlerFunc.ServeHTTP(0xc420162ea0, 0x7fed1a013210, 0xc4202429f0, 0xc42015af00)
	/usr/local/opt/go/libexec/src/net/http/server.go:1947 +0x44
github.com/docker/distribution/health.Handler.func1(0x7fed1a013210, 0xc4202429f0, 0xc42015af00)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/health/health.go:271 +0x123
net/http.HandlerFunc.ServeHTTP(0xc42016d120, 0x7fed1a013210, 0xc4202429f0, 0xc42015af00)
	/usr/local/opt/go/libexec/src/net/http/server.go:1947 +0x44
github.com/docker/distribution/registry.panicHandler.func1(0x7fed1a013210, 0xc4202429f0, 0xc42015af00)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/registry/registry.go:314 +0x81
net/http.HandlerFunc.ServeHTTP(0xc42016d140, 0x7fed1a013210, 0xc4202429f0, 0xc42015af00)
	/usr/local/opt/go/libexec/src/net/http/server.go:1947 +0x44
github.com/docker/distribution/vendor/github.com/gorilla/handlers.combinedLoggingHandler.ServeHTTP(0xe54200, 0xc4200ac008, 0xe55460, 0xc42016d140, 0xe5b840, 0xc4204328c0, 0xc42015af00)
	/Users/hrakers/dev/go/src/github.com/docker/distribution/vendor/github.com/gorilla/handlers/handlers.go:75 +0x123
net/http.serverHandler.ServeHTTP(0xc42009c820, 0xe5b840, 0xc4204328c0, 0xc42015af00)
	/usr/local/opt/go/libexec/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc42036cbe0, 0xe5c380, 0xc420073540)
	/usr/local/opt/go/libexec/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/opt/go/libexec/src/net/http/server.go:2795 +0x27b

This is using the current master 607ae5d on Alpine 3.7 x86_64.

CC @sargun

Metadata

Metadata

Assignees

No one assigned

    Labels

    bugpriority/P0Must have. Very high priority item.

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions