public class ClusterModel
extends java.lang.Object
implements java.io.Serializable
Modifier and Type | Class and Description |
---|---|
static class |
ClusterModel.NonExistentBrokerException
Thrown when a broker is not found in the cluster model
|
Constructor and Description |
---|
ClusterModel(ModelGeneration generation,
double monitoredPartitionsRatio)
Constructor for the cluster class.
|
Modifier and Type | Method and Description |
---|---|
java.util.Set<Broker> |
aliveBrokers()
Get alive brokers in the cluster.
|
java.util.Set<Broker> |
aliveBrokersMatchingAttributes(java.util.Map<java.lang.String,java.lang.String> attributes)
Get brokers with matching attributes
|
java.util.List<Broker> |
aliveBrokersOverThreshold(Resource resource,
double utilizationThreshold) |
java.util.List<Broker> |
aliveBrokersUnderThreshold(Resource resource,
double utilizationThreshold) |
java.util.Set<java.lang.String> |
aliveRackIds() |
java.util.SortedSet<Broker> |
brokenBrokers()
Get broken brokers brokers -- i.e.
|
Broker |
broker(int brokerId)
Get the requested broker in the cluster.
|
java.util.SortedSet<Broker> |
brokers()
Get the set of brokers in the cluster.
|
java.util.Set<Broker> |
brokersHavingOfflineReplicasOnBadDisks()
Get brokers containing offline replicas residing on bad disks in the current cluster model.
|
BrokerStats |
brokerStats(KafkaCruiseControlConfig config)
Get broker return the broker stats.
|
java.util.SortedSet<Broker> |
brokersWithBadDisks()
Get the set of brokers with bad disks -- i.e.
|
java.util.Map<java.lang.Integer,java.lang.String> |
capacityEstimationInfoByBrokerId() |
double |
capacityFor(Resource resource)
Get cluster capacity for the requested resource.
|
void |
clear()
Clear the content and structure of the cluster.
|
void |
clearLoad()
Clear the content of monitoring data at each replica in the cluster.
|
Broker |
createBroker(java.lang.String rackId,
java.lang.String host,
int brokerId,
BrokerCapacityInfo brokerCapacityInfo,
boolean populateReplicaPlacementInfo)
Create a broker under this cluster/rack and get the created broker.
|
void |
createOrDeleteReplicas(java.util.Map<java.lang.Short,java.util.Set<java.lang.String>> topicsByReplicationFactor,
java.util.Map<java.lang.String,java.util.List<java.lang.Integer>> brokersByRack,
java.util.Map<java.lang.Integer,java.lang.String> rackByBroker)
For partitions of specified topics, create or delete replicas in given cluster model to change the partition's replication
factor to target replication factor.
|
Rack |
createRack(java.lang.String rackId)
Create a rack under this cluster.
|
Replica |
createReplica(java.lang.String rackId,
int brokerId,
org.apache.kafka.common.TopicPartition tp,
int index,
boolean isLeader)
Create a replica under given cluster/rack/broker.
|
Replica |
createReplica(java.lang.String rackId,
int brokerId,
org.apache.kafka.common.TopicPartition tp,
int index,
boolean isLeader,
boolean isOffline,
java.lang.String logdir,
boolean isFuture,
boolean isObserver)
Create a replica under given cluster/rack/broker.
|
java.util.SortedSet<Broker> |
deadBrokers()
Get the dead brokers in the cluster.
|
void |
deleteReplica(org.apache.kafka.common.TopicPartition topicPartition,
int brokerId)
Delete a replica from cluster.
|
java.util.SortedSet<Broker> |
demotedBrokers()
Get the demoted brokers in the cluster.
|
ModelGeneration |
generation()
get the metadata generation for this cluster model.
|
ClusterModelStats |
getClusterStats(BalancingConstraint balancingConstraint)
Populate the analysis stats with this cluster and given balancing constraint.
|
java.util.Map<org.apache.kafka.common.TopicPartition,ReplicaPlacementInfo> |
getLeaderDistribution()
Get leader broker ids for each partition.
|
java.util.Map<org.apache.kafka.common.TopicPartition,java.util.List<ReplicaPlacementInfo>> |
getObserverDistribution()
Get the distribution of observer replicas in the cluster.
|
java.util.SortedMap<java.lang.String,java.util.List<Partition>> |
getPartitionsByTopic()
Get a map of partitions by topic names.
|
java.util.Map<org.apache.kafka.common.TopicPartition,java.util.List<ReplicaPlacementInfo>> |
getReplicaDistribution()
Get the distribution of replicas in the cluster at the point of call.
|
kafka.common.TopicPlacement |
getTopicPlacement(java.lang.String topic) |
void |
handleDeadBroker(java.lang.String rackId,
int brokerId,
BrokerCapacityInfo brokerCapacityInfo)
If the rack or broker does not exist, create them with UNKNOWN host name.
|
boolean |
isClusterAlive()
Checks if cluster has at least one alive rack.
|
java.util.Set<Replica> |
leaderReplicas()
Get all the leader replicas in the cluster.
|
Load |
load()
Get the recent cluster load information.
|
int |
maxReplicationFactor()
Get the maximum replication factor of a replica that was added to the cluster before.
|
double |
monitoredPartitionsRatio()
Get the coverage of this cluster model.
|
java.util.SortedSet<Broker> |
newBrokers()
Get the set of new brokers.
|
int |
numAliveRacks()
Get number of alive racks in the cluster.
|
int |
numLeaderReplicas()
Get the number of leader replicas in cluster.
|
int |
numReplicas()
Get the number of replicas in cluster.
|
int |
numTopicReplicas(java.lang.String topic)
Get the number of replicas with the given topic name in cluster.
|
Partition |
partition(org.apache.kafka.common.TopicPartition tp)
Get partition of the given replica.
|
Load |
potentialLeadershipLoadFor(java.lang.Integer brokerId)
Get the leadership load for given broker id.
|
Rack |
rack(java.lang.String rackId)
Get the rack with the rack id if it is found in the cluster; null otherwise.
|
void |
refreshClusterMaxReplicationFactor()
Refresh the maximum topic replication factor statistic.
|
boolean |
relocateLeadership(org.apache.kafka.common.TopicPartition tp,
int sourceBrokerId,
int destinationBrokerId)
(1) Removes leadership from source replica.
|
void |
relocateReplica(org.apache.kafka.common.TopicPartition tp,
int sourceBrokerId,
int destinationBrokerId)
For replica movement across the broker:
(1) Remove the replica from the source broker,
(2) Set the broker of the removed replica as the destination broker,
(3) Add this replica to the destination broker.
|
void |
relocateReplica(org.apache.kafka.common.TopicPartition tp,
int brokerId,
java.lang.String destinationLogdir)
For replica movement across the disks of the same broker.
|
Replica |
removeReplica(int brokerId,
org.apache.kafka.common.TopicPartition tp)
Remove and get removed replica from the cluster.
|
java.util.List<Partition> |
replicasSortedByUtilization(Resource resource,
boolean wantMaxLoad,
boolean wantAvgLoad)
Sort the partitions in the cluster by the utilization of the given resource.
|
java.util.Map<java.lang.String,java.lang.Integer> |
replicationFactorByTopic()
Get the replication factor that each topic in the cluster created with.
|
void |
sanityCheck()
(1) Check whether each load in the cluster contains exactly the number of windows defined by the Load.
|
java.util.Set<Replica> |
selfHealingEligibleReplicas()
Get replicas eligible for self-healing.
|
void |
setBrokerState(int brokerId,
Broker.State newState)
Set the
liveness state of the given broker. |
void |
setReplicaLoad(java.lang.String rackId,
int brokerId,
org.apache.kafka.common.TopicPartition tp,
AggregatedMetricValues metricValues,
java.util.List<java.lang.Long> windows)
Set the load for the given replica.
|
void |
setTopicPlacements(java.util.Map<java.lang.String,kafka.common.TopicPlacement> topicPlacements) |
java.util.List<Broker> |
sortedAliveBrokersUnderThreshold(Resource resource,
double utilizationThreshold)
Get a list of sorted (in ascending order by resource) alive brokers having utilization under:
(given utilization threshold) * (broker and/or host capacity (see
Resource.isHostResource() and
Resource.isBrokerResource() ). |
java.util.Set<java.lang.String> |
topics()
Get topics in the cluster.
|
java.lang.String |
toString() |
void |
trackSortedReplicas(java.lang.String sortName,
java.util.function.Function<Replica,java.lang.Boolean> selectionFunc,
java.util.function.Function<Replica,java.lang.Integer> priorityFunc,
java.util.function.Function<Replica,java.lang.Double> scoreFunc)
Ask the cluster model to keep track of the replicas sorted with the given priority function and score function.
|
void |
trackSortedReplicas(java.lang.String sortName,
java.util.function.Function<Replica,java.lang.Double> scoreFunction)
Ask the cluster model to keep track of the replicas sorted with the given score function.
|
void |
trackSortedReplicas(java.lang.String sortName,
java.util.function.Function<Replica,java.lang.Integer> priorityFunc,
java.util.function.Function<Replica,java.lang.Double> scoreFunc)
Ask the cluster model to keep track of the replicas sorted with the given priority function and score function.
|
boolean |
transferObservership(org.apache.kafka.common.TopicPartition tp,
int sourceBrokerId,
int destinationBrokerId)
Swaps an observer(source) and sync-replica(destination) via the following:
(1) Removes observership from the source replica
(2) Adds observership to the destination replica
(3) Performs a leadership transfer from the destination replica to the source replica if destination replica
is a leader
|
void |
untrackSortedReplicas(java.lang.String sortName)
Untrack the sorted replicas with the given name to release memory.
|
void |
updateReplicationFactor(java.util.Map<java.lang.Short,java.util.Set<java.lang.String>> topicsByReplicationFactor,
java.util.Set<java.lang.Integer> brokersExcludedForReplicaMovement)
For partitions of specified topics, create or delete replicas in given cluster model to change the partition's replication
factor to target replication factor.
|
double[][] |
utilizationMatrix() |
double[] |
variance()
The variance of the derived resources.
|
void |
writeTo(java.io.OutputStream out) |
public ClusterModel(ModelGeneration generation, double monitoredPartitionsRatio)
public ModelGeneration generation()
public double monitoredPartitionsRatio()
public ClusterModelStats getClusterStats(BalancingConstraint balancingConstraint)
balancingConstraint
- Balancing constraint.public java.util.Set<java.lang.String> aliveRackIds()
public Rack rack(java.lang.String rackId)
public java.util.Map<org.apache.kafka.common.TopicPartition,java.util.List<ReplicaPlacementInfo>> getReplicaDistribution()
public java.util.Map<org.apache.kafka.common.TopicPartition,ReplicaPlacementInfo> getLeaderDistribution()
public java.util.Map<org.apache.kafka.common.TopicPartition,java.util.List<ReplicaPlacementInfo>> getObserverDistribution()
public java.util.Set<Replica> selfHealingEligibleReplicas()
public Load load()
public Load potentialLeadershipLoadFor(java.lang.Integer brokerId)
brokerId
- Broker id.public int maxReplicationFactor()
public java.util.Map<java.lang.String,java.lang.Integer> replicationFactorByTopic()
public Partition partition(org.apache.kafka.common.TopicPartition tp)
tp
- Topic partition of the replica for which the partition is requested.public java.util.SortedMap<java.lang.String,java.util.List<Partition>> getPartitionsByTopic()
public java.util.Set<Replica> leaderReplicas()
public void setBrokerState(int brokerId, Broker.State newState) throws ClusterModel.NonExistentBrokerException
liveness state
of the given broker.
brokerId
- Id of the broker for which the alive status is set.newState
- The new state of the broker.ClusterModel.NonExistentBrokerException
public void relocateReplica(org.apache.kafka.common.TopicPartition tp, int brokerId, java.lang.String destinationLogdir)
tp
- Partition Info of the replica to be relocated.brokerId
- Broker id.destinationLogdir
- Destination logdir.public void relocateReplica(org.apache.kafka.common.TopicPartition tp, int sourceBrokerId, int destinationBrokerId)
tp
- Partition Info of the replica to be relocated.sourceBrokerId
- Source broker id.destinationBrokerId
- Destination broker id.public boolean relocateLeadership(org.apache.kafka.common.TopicPartition tp, int sourceBrokerId, int destinationBrokerId)
tp
- Topic partition of this replica.sourceBrokerId
- Source broker id.destinationBrokerId
- Destination broker id.public boolean transferObservership(org.apache.kafka.common.TopicPartition tp, int sourceBrokerId, int destinationBrokerId)
public java.util.Set<Broker> aliveBrokers()
public java.util.SortedSet<Broker> deadBrokers()
public java.util.SortedSet<Broker> brokenBrokers()
public java.util.Map<java.lang.Integer,java.lang.String> capacityEstimationInfoByBrokerId()
public java.util.SortedSet<Broker> demotedBrokers()
public java.util.SortedSet<Broker> newBrokers()
public java.util.SortedSet<Broker> brokersWithBadDisks()
public java.util.Set<Broker> brokersHavingOfflineReplicasOnBadDisks()
public java.util.Set<Broker> aliveBrokersMatchingAttributes(java.util.Map<java.lang.String,java.lang.String> attributes)
public boolean isClusterAlive()
public void clearLoad()
public Replica removeReplica(int brokerId, org.apache.kafka.common.TopicPartition tp)
brokerId
- Id of the broker containing the partition.tp
- Topic partition of the replica to be removed.public java.util.SortedSet<Broker> brokers()
public Broker broker(int brokerId)
brokerId
- Id of the requested broker.public void trackSortedReplicas(java.lang.String sortName, java.util.function.Function<Replica,java.lang.Double> scoreFunction)
ReplicaSortFunctionFactory
so the functions can be maintained
in a single place.
The sorted replica will only be updated in the following cases:
1. A replica is added to or removed from a broker
2. A replica's role has changed from leader to follower, and vice versa.
The sorted replicas are named using the given sortName, and can be accessed using
Broker.trackedSortedReplicas(String)
. If the sorted replicas are no longer needed,
untrackSortedReplicas(String)
to release memory.sortName
- the name of the sorted replicas.scoreFunction
- the score function to sort the replicas with the same priority, replicas are sorted in ascending
order of score.SortedReplicas
public void trackSortedReplicas(java.lang.String sortName, java.util.function.Function<Replica,java.lang.Integer> priorityFunc, java.util.function.Function<Replica,java.lang.Double> scoreFunc)
ReplicaSortFunctionFactory
so the functions can be maintained
in a single place.
The sort will first use the priority function then the score function. The priority function allows the
caller to prioritize a certain type of replicas, e.g immigrant replicas.
The sorted replica will only be updated in the following cases:
1. A replica is added to or removed from abroker
2. A replica's role has changed from leader to follower, and vice versa.
The sorted replicas are named using the given sortName, and can be accessed using
Broker.trackedSortedReplicas(String)
. If the sorted replicas are no longer needed,
untrackSortedReplicas(String)
to release memory.sortName
- the name of the sorted replicas.priorityFunc
- the priority function to sort the replicasscoreFunc
- the score function to sort the replicas with the same priority, replicas are sorted in ascending
order of score.SortedReplicas
public void trackSortedReplicas(java.lang.String sortName, java.util.function.Function<Replica,java.lang.Boolean> selectionFunc, java.util.function.Function<Replica,java.lang.Integer> priorityFunc, java.util.function.Function<Replica,java.lang.Double> scoreFunc)
ReplicaSortFunctionFactory
so the functions can be maintained
in a single place.
The sorted replica will only be updated in the following cases:
1. A replica is added to or removed from a broker
2. A replica's role has changed from leader to follower, and vice versa.
The sorted replicas are named using the given sortName, and can be accessed using
Broker.trackedSortedReplicas(String)
. If the sorted replicas are no longer needed,
untrackSortedReplicas(String)
to release memory.sortName
- the name of the sorted replicas.selectionFunc
- the selection function to decide which replicas to include in the sort. If it is null
,
all the replicas are to be included.priorityFunc
- the priority function to sort the replicasscoreFunc
- the score function to sort the replicas with the same priority, replicas are sorted in ascending
order of score.SortedReplicas
public void untrackSortedReplicas(java.lang.String sortName)
sortName
- the name of the sorted replicas.public void clear()
public int numAliveRacks()
public int numTopicReplicas(java.lang.String topic)
topic
- Name of the topic for which the number of replicas in cluster will be counted.public int numLeaderReplicas()
public int numReplicas()
public java.util.Set<java.lang.String> topics()
public double capacityFor(Resource resource)
resource
- Resource for which the capacity will be provided.public void setReplicaLoad(java.lang.String rackId, int brokerId, org.apache.kafka.common.TopicPartition tp, AggregatedMetricValues metricValues, java.util.List<java.lang.Long> windows)
rackId
- Rack id.brokerId
- Broker Id containing the replica with the given topic partition.tp
- Topic partition that identifies the replica in this broker.metricValues
- The load of the replica.windows
- The windows list of the aggregated metrics.public void handleDeadBroker(java.lang.String rackId, int brokerId, BrokerCapacityInfo brokerCapacityInfo)
rackId
- Rack id under which the replica will be created.brokerId
- Broker id under which the replica will be created.brokerCapacityInfo
- The capacity information to use if the broker does not exist.public Replica createReplica(java.lang.String rackId, int brokerId, org.apache.kafka.common.TopicPartition tp, int index, boolean isLeader)
LoadMonitor
uses createReplica(String, int,
TopicPartition, int, boolean, boolean, String, boolean, boolean)
while setting the replica offline status, and it
considers the broken disks as well. Whereas, this method is used only by the unit tests. The relevant unit tests
may use Replica.markOriginalOffline()
to mark offline replicas on broken disks.
The main reason for this separation is the lack of disk representation in the current broker model. Once the
patch #327 is merged, we can simplify this logic.rackId
- Rack id under which the replica will be created.brokerId
- Broker id under which the replica will be created.tp
- Topic partition information of the replica.index
- The index of the replica in the replica list.isLeader
- True if the replica is a leader, false otherwise.public Replica createReplica(java.lang.String rackId, int brokerId, org.apache.kafka.common.TopicPartition tp, int index, boolean isLeader, boolean isOffline, java.lang.String logdir, boolean isFuture, boolean isObserver)
rackId
- Rack id under which the replica will be created.brokerId
- Broker id under which the replica will be created.tp
- Topic partition information of the replica.index
- The index of the replica in the replica list.isLeader
- True if the replica is a leader, false otherwise.isOffline
- True if the replica is offline in its original location, false otherwise.logdir
- The logdir of replica's hosting disk. If replica placement over disk information is not populated,
this parameter is null.isFuture
- True if the replica does not correspond to any existing replica in the cluster, but a replica
we are going to add to the cluster. This replica's original broker will not be any existing broker
so that it will be treated as an immigrant replica for whatever broker it is assigned to and
grant goals greatest freedom to allocate to an existing broker.isObserver
- True if the replica is an observer, false otherwisepublic void deleteReplica(org.apache.kafka.common.TopicPartition topicPartition, int brokerId)
refreshClusterMaxReplicationFactor()
after all replica deletion.topicPartition
- Topic partition of the replica to be removed.brokerId
- Id of the broker hosting the replica.public void refreshClusterMaxReplicationFactor()
public Broker createBroker(java.lang.String rackId, java.lang.String host, int brokerId, BrokerCapacityInfo brokerCapacityInfo, boolean populateReplicaPlacementInfo)
_capacityEstimationInfoByBrokerId
if the broker capacity has been estimated.rackId
- Id of the rack that the broker will be created in.host
- The host of this brokerbrokerId
- Id of the broker to be created.brokerCapacityInfo
- Capacity information of the created broker.populateReplicaPlacementInfo
- Whether populate replica placement over disk information or not.public Rack createRack(java.lang.String rackId)
rackId
- Id of the rack to be created.public void createOrDeleteReplicas(java.util.Map<java.lang.Short,java.util.Set<java.lang.String>> topicsByReplicationFactor, java.util.Map<java.lang.String,java.util.List<java.lang.Integer>> brokersByRack, java.util.Map<java.lang.Integer,java.lang.String> rackByBroker)
topicsByReplicationFactor
- The topics to modify replication factor with target replication factor.brokersByRack
- A map from rack to broker.rackByBroker
- A map from broker to rack.public void updateReplicationFactor(java.util.Map<java.lang.Short,java.util.Set<java.lang.String>> topicsByReplicationFactor, java.util.Set<java.lang.Integer> brokersExcludedForReplicaMovement)
topicsByReplicationFactor
- The topics to modify replication factor with target replication factor.public java.util.List<Broker> sortedAliveBrokersUnderThreshold(Resource resource, double utilizationThreshold)
Resource.isHostResource()
and
Resource.isBrokerResource()
). Utilization threshold might be any capacity constraint thresholds such as
balance or capacity.resource
- Resource for which brokers will be sorted.utilizationThreshold
- Utilization threshold for the given resource.public java.util.List<Broker> aliveBrokersUnderThreshold(Resource resource, double utilizationThreshold)
public java.util.List<Broker> aliveBrokersOverThreshold(Resource resource, double utilizationThreshold)
public java.util.List<Partition> replicasSortedByUtilization(Resource resource, boolean wantMaxLoad, boolean wantAvgLoad)
resource
- the resource type.wantMaxLoad
- True if the requested utilization represents the peak load, false otherwise.wantAvgLoad
- True if the requested utilization represents the avg load, false otherwise.public void sanityCheck()
public BrokerStats brokerStats(KafkaCruiseControlConfig config)
public double[] variance()
public double[][] utilizationMatrix()
public void writeTo(java.io.OutputStream out) throws java.io.IOException
java.io.IOException
public kafka.common.TopicPlacement getTopicPlacement(java.lang.String topic)
public void setTopicPlacements(java.util.Map<java.lang.String,kafka.common.TopicPlacement> topicPlacements)
public java.lang.String toString()
toString
in class java.lang.Object