Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions kafka/coordinator/assignors/roundrobin.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,15 @@ def assign(cls, cluster, members):
# construct {member_id: {topic: [partition, ...]}}
assignment = collections.defaultdict(lambda: collections.defaultdict(list))

# Sort static and dynamic members separately to maintain stable static assignments
ungrouped = [(member.group_instance_id, member.member_id) for member in members]
grouped = {k: list(g) for k, g in itertools.groupby(ungrouped, key=lambda ids: ids[0] is not None)}
member_list = sorted(grouped.get(True, [])) + sorted(grouped.get(False, [])) # sorted static members first, then sorted dynamic
member_iter = itertools.cycle(member_list)
# Static members sort by instance ID first, followed by dynamic members.
member_list = sorted(members, key=lambda member: (
member.group_instance_id is None,
member.group_instance_id or '', member.member_id))
member_iter = itertools.cycle(member.member_id for member in member_list)
member_topics = {member.member_id: member.metadata.topics for member in members}

for partition in all_topic_partitions:
_group_instance_id, member_id = next(member_iter)
member_id = next(member_iter)

# Because we constructed all_topic_partitions from the set of
# member subscribed topics, we should be safe assuming that
Expand Down
38 changes: 38 additions & 0 deletions test/consumer/test_assignors.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from random import randint, sample

import pytest
from itertools import permutations

from kafka.structs import TopicPartition
from kafka.coordinator.assignors.abstract import ConsumerProtocolAssignment
Expand Down Expand Up @@ -1051,3 +1052,40 @@ def test_assign_returns_assignments_for_all_members(self, mocker):
cluster = create_cluster(mocker, {'t'}, topics_partitions={0, 1})
ret = assignor.assign(cluster, members)
assert set(ret) == {'C0', 'C1'}


@pytest.mark.parametrize('order', list(permutations(range(3))))
@pytest.mark.parametrize('instances,expected', [
((None, None, None), {'C0': [('t0', [0])], 'C1': [('t1', [0])],
'C2': [('t1', [1]), ('t2', [0, 1, 2])]}),
(('z', 'a', 'm'), {'C0': [], 'C1': [('t0', [0]), ('t1', [1])],
'C2': [('t1', [0]), ('t2', [0, 1, 2])]}),
(('z', None, 'a'), {'C0': [], 'C1': [('t1', [0])],
'C2': [('t0', [0]), ('t1', [1]), ('t2', [0, 1, 2])]}),
])
def test_roundrobin_differing_subscriptions(mocker, order, instances, expected):
subscriptions = [('t0',), ('t0', 't1'), ('t0', 't1', 't2')]
members = [JoinGroupResponse.JoinGroupResponseMember(
member_id='C%d' % i, group_instance_id=instances[i],
metadata=RoundRobinPartitionAssignor.metadata(subscriptions[i]))
for i in order]
partitions = {'t0': {0}, 't1': {0, 1}, 't2': {0, 1, 2}}
cluster = create_cluster(mocker, set(partitions),
topic_partitions_lambda=partitions.get)
result = RoundRobinPartitionAssignor.assign(cluster, members)
assert set(result) == set(expected)
for member_id, assignment in expected.items():
assert result[member_id].assignment == assignment
assert ConsumerProtocolAssignment.decode(result[member_id].encode()) == result[member_id]


def test_roundrobin_missing_metadata_and_idle_members(mocker):
members = make_join_group_response_members({
'C0': RoundRobinPartitionAssignor.metadata({'active', 'missing'}),
'C1': RoundRobinPartitionAssignor.metadata(set()),
})
cluster = create_cluster(mocker, {'active', 'missing'},
topic_partitions_lambda={'active': {0}}.get)
assignments = RoundRobinPartitionAssignor.assign(cluster, members)
assert assignments['C0'].assignment == [('active', [0])]
assert assignments['C1'].assignment == []
77 changes: 77 additions & 0 deletions test/consumer/test_coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2883,3 +2883,80 @@ def test_metadata_growth_triggers_rejoin_end_to_end(net, metrics):
coordinator.close(timeout_ms=0)
client.close()
manager.close()


@pytest.mark.parametrize('instances,order,expected', [
((None, None, None), (2, 0, 1),
{'C0': [('t0', [0])], 'C1': [('t1', [0])],
'C2': [('t1', [1]), ('t2', [0, 1, 2])]}),
(('z', 'a', 'm'), (0, 2, 1),
{'C0': [], 'C1': [('t0', [0]), ('t1', [1])],
'C2': [('t1', [0]), ('t2', [0, 1, 2])]}),
(('z', None, 'a'), (0, 1, 2),
{'C0': [], 'C1': [('t1', [0])],
'C2': [('t0', [0]), ('t1', [1]), ('t2', [0, 1, 2])]}),
])
def test_perform_roundrobin_assignment_with_different_subscriptions(
coordinator, instances, order, expected):
coordinator._subscription.subscribe(topics=['t0'])
Broker = MetadataResponse.MetadataResponseBroker
Topic = MetadataResponse.MetadataResponseTopic
Partition = Topic.MetadataResponsePartition
response = MetadataResponse[0](
brokers=[Broker(0, 'localhost', 9092, version=0)],
topics=[Topic(0, 't%d' % topic,
[Partition(0, part, 0, [0], [0], version=0)
for part in range(topic + 1)], version=0)
for topic in range(3)])
coordinator._client.cluster.update_metadata(
MetadataResponse.decode(response.encode(), version=0))
members = [JoinGroupResponse.JoinGroupResponseMember(
member_id='C%d' % i, group_instance_id=instances[i],
metadata=ConsumerProtocolSubscription(
0, ['t%d' % topic for topic in range(i + 1)], b'').encode())
for i in order]
assignments = coordinator._perform_assignment('C0', 'roundrobin', members)
assert {member: assignment.assignment for member, assignment in assignments.items()} == expected
assert coordinator._subscription.group_subscription() == {'t0', 't1', 't2'}
for assignment in assignments.values():
assert ConsumerProtocolAssignment.decode(assignment.encode()) == assignment


def test_roundrobin_join_sends_differing_subscription_assignments(broker, seeded_coord):
Topic = MetadataResponse.MetadataResponseTopic
Partition = Topic.MetadataResponsePartition
topics = [Topic(error_code=0, name='t%d' % topic, is_internal=False,
partitions=[Partition(error_code=0, partition_index=part,
leader_id=0, leader_epoch=0,
replica_nodes=[0], isr_nodes=[0],
offline_replicas=[])
for part in range(topic + 1)])
for topic in range(3)]
broker.set_metadata(topics=topics)
metadata = broker._metadata_response
seeded_coord._client.cluster.update_metadata(
MetadataResponse.decode(metadata.encode(), version=metadata._version))
seeded_coord._subscription.subscribe(topics=['t0'])
members = [JoinGroupResponse.JoinGroupResponseMember(
member_id='C%d' % i, group_instance_id=instance,
metadata=ConsumerProtocolSubscription(
0, ['t%d' % topic for topic in range(i + 1)], b'').encode())
for i, instance in enumerate(('z', None, 'a'))]
broker.respond(JoinGroupRequest, _join_response_object(
leader='C0', member_id='C0', protocol_name='roundrobin', members=members))
captured = {}
expected = {'C0': [], 'C1': [('t1', [0])],
'C2': [('t0', [0]), ('t1', [1]), ('t2', [0, 1, 2])]}

def sync_handler(api_key, api_version, correlation_id, request_bytes):
request = SyncGroupRequest.decode(request_bytes, version=api_version, header=True)
captured.update({member.member_id: ConsumerProtocolAssignment.decode(member.assignment).assignment
for member in request.assignments})
return _sync_response_object(
protocol_name='roundrobin',
assignment=ConsumerProtocolAssignment(0, expected['C0'], b'').encode())

broker.respond_fn(SyncGroupRequest, sync_handler)
result = seeded_coord._net.run(seeded_coord._do_join_and_sync_async)
assert captured == expected
assert ConsumerProtocolAssignment.decode(result).assignment == expected['C0']