Found the bug:

akka.net/src/core/Akka.Cluster/ClusterDaemon.cs

Lines 2145 to 2176 in 881019e

    
           private void MoveJoiningToWeaklyUp() 
        
           { 
        
               var localGossip = _latestGossip; 
        
               var localMembers = localGossip.Members; 
        
               var enoughMembers = IsMinNrOfMembersFulfilled(); 
        
               bool IsJoiningToWeaklyUp(Member m) => m.Status == MemberStatus.Joining 
        
                                                     && enoughMembers 
        
                                                     && _latestGossip.ReachabilityExcludingDownedObservers.Value.IsReachable(m.UniqueAddress); 
        
               var changedMembers = localMembers 
        
                   .Where(IsJoiningToWeaklyUp) 
        
                   .Select(m => m.Copy(MemberStatus.WeaklyUp)) 
        
                   .ToImmutableSortedSet(); 
        
               if (!changedMembers.IsEmpty) 
        
               { 
        
                   // replace changed members 
        
                   var newMembers = changedMembers.Union(localMembers); 
        
                   var newGossip = localGossip.Copy(members: newMembers); 
        
                   UpdateLatestGossip(newGossip); 
        
                   // log status change 
        
                   foreach (var m in changedMembers) 
        
                   { 
        
                       _log.Info("Leader is moving node [{0}] to [{1}]", m.Address, m.Status); 
        
                   } 
        
                   Publish(newGossip); 
        
                   if (_cluster.Settings.PublishStatsInterval == TimeSpan.Zero) PublishInternalStats(); 
        
               } 
        
           }

This, specifically, is the trouble-maker:

var newMembers = changedMembers.Union(localMembers);

FYI @Horusiath, when we make changes to the membership of the cluster we can't rely on consistent left-to-right set semantics from the built-in System.Collections.Immutable classes. They don't work like how they do on the JVM and will use performance optimizations to change around which copy of the "member" they might actually use. I went into some detail on why this is here: #2584 (comment)

We need to use the PickNextTransition function instead:

akka.net/src/core/Akka.Cluster/ClusterDaemon.cs

Lines 2284 to 2287 in 881019e

    
           var newMembers = Member.PickNextTransition(changedMembers, localMembers) 
        
               .Except(removedUnreachable) 
        
               .Where(x => !removedExitingConfirmed.Contains(x.UniqueAddress)) 
        
               .ToImmutableSortedSet();

	private void MoveJoiningToWeaklyUp()
	{
	var localGossip = _latestGossip;
	var localMembers = localGossip.Members;
	var enoughMembers = IsMinNrOfMembersFulfilled();

	bool IsJoiningToWeaklyUp(Member m) => m.Status == MemberStatus.Joining
	&& enoughMembers
	&& _latestGossip.ReachabilityExcludingDownedObservers.Value.IsReachable(m.UniqueAddress);

	var changedMembers = localMembers
	.Where(IsJoiningToWeaklyUp)
	.Select(m => m.Copy(MemberStatus.WeaklyUp))
	.ToImmutableSortedSet();

	if (!changedMembers.IsEmpty)
	{
	// replace changed members
	var newMembers = changedMembers.Union(localMembers);
	var newGossip = localGossip.Copy(members: newMembers);
	UpdateLatestGossip(newGossip);

	// log status change
	foreach (var m in changedMembers)
	{
	_log.Info("Leader is moving node [{0}] to [{1}]", m.Address, m.Status);
	}

	Publish(newGossip);
	if (_cluster.Settings.PublishStatsInterval == TimeSpan.Zero) PublishInternalStats();
	}
	}

	var newMembers = Member.PickNextTransition(changedMembers, localMembers)
	.Except(removedUnreachable)
	.Where(x => !removedExitingConfirmed.Contains(x.UniqueAddress))
	.ToImmutableSortedSet();

WeaklyUp nodes get stuck at joining #3274

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions