Skip to content

Support a domain qualified finalizer instead of one that triggers a warning from kubernetes #7273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 18, 2024
Merged

Support a domain qualified finalizer instead of one that triggers a warning from kubernetes #7273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
de85bd7
Fix config api defaults updated messages
jsoref Sep 13, 2024
6564f1c
spelling: feature
jsoref Sep 13, 2024
0afe4ab
Add support for domain qualified finalizer
jsoref Sep 13, 2024
4d97c4c
Add feature gate for domain qualified finalizer
jsoref Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/apis/config/controller/v1alpha1/defaults_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func TestControllerConfigurationDefaults(t *testing.T) {
if err := os.WriteFile(tt.jsonFilePath, defaultData, 0644); err != nil {
t.Fatal(err)
}
t.Log("cainjector config api defaults updated")
t.Log("controller config api defaults updated")
}

expectedData, err := os.ReadFile(tt.jsonFilePath)
Expand Down
2 changes: 1 addition & 1 deletion internal/apis/config/webhook/v1alpha1/defaults_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func TestWebhookConfigurationDefaults(t *testing.T) {
if err := os.WriteFile(tt.jsonFilePath, defaultData, 0644); err != nil {
t.Fatal(err)
}
t.Log("cainjector config api defaults updated")
t.Log("webhook config api defaults updated")
}

expectedData, err := os.ReadFile(tt.jsonFilePath)
Expand Down
2 changes: 1 addition & 1 deletion internal/cainjector/feature/features.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ limitations under the License.

// feature contains cainjector feature gate setup code. Do not import this
// package into any code that's shared with other components to prevent
// overwriting other component's featue gates, see i.e
// overwriting other component's feature gates, see i.e
// https://github.com/cert-manager/cert-manager/issues/6011
package feature

Expand Down
11 changes: 10 additions & 1 deletion internal/controller/feature/features.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ limitations under the License.

// feature contains controller's feature gate setup functionality. Do not import
// this package into any code that's shared with other components to prevent
// overwriting other component's featue gates, see i.e
// overwriting other component's feature gates, see i.e
// https://github.com/cert-manager/cert-manager/issues/6011
package feature

Expand Down Expand Up @@ -137,6 +137,14 @@ const (
// Certificate resources.
// Github Issue: https://github.com/cert-manager/cert-manager/issues/6393
OtherNames featuregate.Feature = "OtherNames"

// Owner: @jsoref
// Alpha: v1.16
//
// UseDomainQualifiedFinalizer changes the finalizer added to cert-manager created
// resources to acme.cert-manager.io/finalizer instead of finalizer.acme.cert-manager.io.
// GitHub Issue: https://github.com/cert-manager/cert-manager/issues/7266
UseDomainQualifiedFinalizer featuregate.Feature = "UseDomainQualifiedFinalizer"
)

func init() {
Expand All @@ -160,4 +168,5 @@ var defaultCertManagerFeatureGates = map[featuregate.Feature]featuregate.Feature
UseCertificateRequestBasicConstraints: {Default: false, PreRelease: featuregate.Alpha},
NameConstraints: {Default: false, PreRelease: featuregate.Alpha},
OtherNames: {Default: false, PreRelease: featuregate.Alpha},
UseDomainQualifiedFinalizer: {Default: false, PreRelease: featuregate.Alpha},
}
2 changes: 1 addition & 1 deletion internal/webhook/feature/features.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ limitations under the License.

// feature contains webhook's feature gate setup functionality. Do not import
// this package into any code that's shared with other components to prevent
// overwriting other component's featue gates, see i.e
// overwriting other component's feature gates, see i.e
// https://github.com/cert-manager/cert-manager/issues/6011
package feature

Expand Down
3 changes: 2 additions & 1 deletion pkg/apis/acme/v1/const.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,6 @@ limitations under the License.
package v1

const (
ACMEFinalizer = "finalizer.acme.cert-manager.io"
ACMELegacyFinalizer = "finalizer.acme.cert-manager.io"
ACMEDomainQualifiedFinalizer = "acme.cert-manager.io/finalizer"
)
14 changes: 13 additions & 1 deletion pkg/controller/acmechallenges/finalizer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,18 @@ import (
// allowing the garbage collector to remove the challenge.

// finalizerRequired returns true if the finalizer is not found on the challenge.
//
// API transition
// We currently only add cmacme.ACMELegacyFinalizer, but a future version will add
// cmacme.ACMEDomainQualifiedFinalizer.
// A finalizer only needs to be added if neither is present.
func finalizerRequired(ch *cmacme.Challenge) bool {
return !sets.NewString(ch.Finalizers...).Has(cmacme.ACMEFinalizer)
finalizers := sets.NewString(ch.Finalizers...)
return !finalizers.Has(cmacme.ACMELegacyFinalizer) &&
!finalizers.Has(cmacme.ACMEDomainQualifiedFinalizer)
}

func otherFinalizerPresent(ch *cmacme.Challenge) bool {
return ch.Finalizers[0] != cmacme.ACMELegacyFinalizer &&
ch.Finalizers[0] != cmacme.ACMEDomainQualifiedFinalizer
}
28 changes: 24 additions & 4 deletions pkg/controller/acmechallenges/finalizer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,33 @@ func Test_finalizerRequired(t *testing.T) {
want: true,
},
{
name: "only-native-finalizer",
finalizers: []string{cmacme.ACMEFinalizer},
name: "only-native-legacy-finalizer",
finalizers: []string{cmacme.ACMELegacyFinalizer},
want: false,
},
{
name: "some-foreign-finalizers",
finalizers: []string{"f1", "f2", cmacme.ACMEFinalizer, "f3"},
name: "only-native-domain-qualified-finalizer",
finalizers: []string{cmacme.ACMEDomainQualifiedFinalizer},
want: false,
},
{
name: "both-native-finalizers",
finalizers: []string{cmacme.ACMELegacyFinalizer, cmacme.ACMEDomainQualifiedFinalizer},
want: false,
},
{
name: "some-foreign-and-legacy-finalizer",
finalizers: []string{"f1", "f2", cmacme.ACMELegacyFinalizer, "f3"},
want: false,
},
{
name: "some-foreign-and-domain-qualified-finalizer",
finalizers: []string{"f1", "f2", cmacme.ACMEDomainQualifiedFinalizer, "f3"},
want: false,
},
{
name: "some-foreign-and-legacy-finalizer-and-domain-qualified-finalizer",
finalizers: []string{"f1", "f2", cmacme.ACMELegacyFinalizer, cmacme.ACMEDomainQualifiedFinalizer, "f3"},
want: false,
},
{
Expand Down
21 changes: 18 additions & 3 deletions pkg/controller/acmechallenges/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"errors"
"fmt"
"slices"
"time"

acmeapi "golang.org/x/crypto/acme"
Expand Down Expand Up @@ -92,8 +93,20 @@ func (c *controller) Sync(ctx context.Context, chOriginal *cmacme.Challenge) (er
// This finalizer ensures that the challenge is not garbage collected before
// cert-manager has a chance to clean up resources created for the
// challenge.
//
// API Transition
// -- Until UseDomainQualifiedFinalizer is active, we add cmacme.ACMELegacyFinalizer.
// -- When it is active we add cmacme.ACMEDomainQualifiedFinalizer instead.
//
// -- Both finalizers are supported, the flag just controls the one we add.
//
// -- We only need to add a finalizer label if no supported finalizer label is present.
if finalizerRequired(ch) {
ch.Finalizers = append(ch.Finalizers, cmacme.ACMEFinalizer)
finalizer := cmacme.ACMELegacyFinalizer
if utilfeature.DefaultFeatureGate.Enabled(feature.UseDomainQualifiedFinalizer) {
finalizer = cmacme.ACMEDomainQualifiedFinalizer
}
ch.Finalizers = append(ch.Finalizers, finalizer)
return nil
}

Expand Down Expand Up @@ -256,14 +269,16 @@ func (c *controller) handleFinalizer(ctx context.Context, ch *cmacme.Challenge)
if len(ch.Finalizers) == 0 {
return nil
}
if ch.Finalizers[0] != cmacme.ACMEFinalizer {
if otherFinalizerPresent(ch) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This logic looks odd to me. Why are we just checking the first finalizer now? And I don't think we need the otherFinalizerPresent function. Why do we care if the resource has another finalizer here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The current code is black magic. There is a model of naming functions to clarify the intent of black magic "comments as code" . The alternative model is to add comments which regularly fall out of sync.

I'm guessing about the intent of the current code and trying to generalize it to the case that one or both of the supported finalizers are present.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@munnerz added the relevant code in #1116

dd8f987

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand the need for this logic. Kubernetes will only finalize the resource deletion when all finalizers are gone: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/. We should only have to care for "our" finalizers. If a foreign controller has added a finalizer, that controller should remove its finalizer.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only reason I can think of is that the ordering would be important, but I doubt that is the case.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Assuming the current code is wrong, what should it be doing?

Is it trying to see if any other finalizer is in any slot of the list? Or is it trying to see if our finalizer isn't in the list? Or should the logic be removed?

The code change here is the closest to the spirit of the current code, but it sure doesn't seem like the current behavior is desired.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree, your change matches the current behavior.
From what I understand this check makes sure that the ACME finalizer is the first finalizer in the list, otherwise it returns with a nil error.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@munnerz do you remember why you added this check?

Copy link
Member

@erikgb erikgb Sep 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would collapse this conditional with the previous one: Exit early if none of our finalizers are present.

if finalizerRequired(ch) {
  return nil
}

Maybe @munnerz remembers why?

log.V(logf.DebugLevel).Info("waiting to run challenge finalization...")
return nil
}

defer func() {
// call Update to remove the metadata.finalizers entry
ch.Finalizers = ch.Finalizers[1:]
ch.Finalizers = slices.DeleteFunc(ch.Finalizers, func(finalizer string) bool {
return finalizer == cmacme.ACMELegacyFinalizer || finalizer == cmacme.ACMEDomainQualifiedFinalizer
})
}()

if !ch.Status.Processing {
Expand Down
29 changes: 26 additions & 3 deletions pkg/controller/acmechallenges/sync_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,17 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
coretesting "k8s.io/client-go/testing"
featuregatetesting "k8s.io/component-base/featuregate/testing"

"github.com/cert-manager/cert-manager/internal/controller/feature"
accountstest "github.com/cert-manager/cert-manager/pkg/acme/accounts/test"
acmecl "github.com/cert-manager/cert-manager/pkg/acme/client"
cmacme "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
v1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
testpkg "github.com/cert-manager/cert-manager/pkg/controller/test"
"github.com/cert-manager/cert-manager/pkg/issuer"
utilfeature "github.com/cert-manager/cert-manager/pkg/util/feature"
"github.com/cert-manager/cert-manager/test/unit/gen"
)

Expand Down Expand Up @@ -71,7 +74,7 @@ type testT struct {
acmeClient *acmecl.FakeACME
}

func TestSyncHappyPath(t *testing.T) {
func testSyncHappyPathWithFinalizer(t *testing.T, finalizer string, activeFinalizer string) {
testIssuerHTTP01Enabled := gen.Issuer("testissuer", gen.SetIssuerACME(cmacme.ACMEIssuer{
Solvers: []cmacme.ACMEChallengeSolver{
{
Expand All @@ -85,7 +88,7 @@ func TestSyncHappyPath(t *testing.T) {
gen.SetChallengeIssuer(cmmeta.ObjectReference{
Name: "testissuer",
}),
gen.SetChallengeFinalizers([]string{cmacme.ACMEFinalizer}),
gen.SetChallengeFinalizers([]string{finalizer}),
)
deletedChallenge := gen.ChallengeFrom(baseChallenge,
gen.SetChallengeDeletionTimestamp(metav1.Now()))
Expand Down Expand Up @@ -188,7 +191,7 @@ func TestSyncHappyPath(t *testing.T) {
gen.DefaultTestNamespace,
gen.ChallengeFrom(baseChallenge,
gen.SetChallengeProcessing(true),
gen.SetChallengeFinalizers([]string{cmacme.ACMEFinalizer})))),
gen.SetChallengeFinalizers([]string{activeFinalizer})))),
},
},
expectErr: false,
Expand Down Expand Up @@ -588,6 +591,26 @@ func TestSyncHappyPath(t *testing.T) {
}
}

func TestSyncHappyPathFinalizerLegacyToLegacy(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, feature.UseDomainQualifiedFinalizer, false)
testSyncHappyPathWithFinalizer(t, cmacme.ACMELegacyFinalizer, cmacme.ACMELegacyFinalizer)
}

func TestSyncHappyPathFinalizerDomainQualifiedToLegacy(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, feature.UseDomainQualifiedFinalizer, false)
testSyncHappyPathWithFinalizer(t, cmacme.ACMEDomainQualifiedFinalizer, cmacme.ACMELegacyFinalizer)
}

func TestSyncHappyPathFinalizerLegacyToDomainQualified(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, feature.UseDomainQualifiedFinalizer, true)
testSyncHappyPathWithFinalizer(t, cmacme.ACMELegacyFinalizer, cmacme.ACMEDomainQualifiedFinalizer)
}

func TestSyncHappyPathFinalizerDomainQualifiedToDomainQualified(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, feature.UseDomainQualifiedFinalizer, true)
testSyncHappyPathWithFinalizer(t, cmacme.ACMEDomainQualifiedFinalizer, cmacme.ACMEDomainQualifiedFinalizer)
}

func runTest(t *testing.T, test testT) {
test.builder.T = t
test.builder.Init()
Expand Down