-
Notifications
You must be signed in to change notification settings - Fork 14.8k
MINOR: Initialize fetchPartitionStatus as a Map type to reduce collection conversions #20768
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,6 +30,7 @@ import org.apache.kafka.server.purgatory.DelayedOperation | |
| import org.apache.kafka.server.storage.log.{FetchIsolation, FetchParams, FetchPartitionData} | ||
| import org.apache.kafka.storage.internals.log.{FetchPartitionStatus, LogOffsetMetadata} | ||
|
|
||
| import java.util | ||
| import scala.collection._ | ||
| import scala.jdk.CollectionConverters._ | ||
|
|
||
|
|
@@ -39,7 +40,7 @@ import scala.jdk.CollectionConverters._ | |
| */ | ||
| class DelayedFetch( | ||
| params: FetchParams, | ||
| fetchPartitionStatus: Seq[(TopicIdPartition, FetchPartitionStatus)], | ||
| fetchPartitionStatus: util.LinkedHashMap[TopicIdPartition, FetchPartitionStatus], | ||
| replicaManager: ReplicaManager, | ||
| quota: ReplicaQuota, | ||
| responseCallback: Seq[(TopicIdPartition, FetchPartitionData)] => Unit | ||
|
|
@@ -66,8 +67,7 @@ class DelayedFetch( | |
| */ | ||
| override def tryComplete(): Boolean = { | ||
| var accumulatedSize = 0 | ||
| fetchPartitionStatus.foreach { | ||
| case (topicIdPartition, fetchStatus) => | ||
| fetchPartitionStatus.forEach { (topicIdPartition, fetchStatus) => | ||
| val fetchOffset = fetchStatus.startOffsetMetadata | ||
| val fetchLeaderEpoch = fetchStatus.fetchInfo.currentLeaderEpoch | ||
| try { | ||
|
|
@@ -154,9 +154,9 @@ class DelayedFetch( | |
| * Upon completion, read whatever data is available and pass to the complete callback | ||
| */ | ||
| override def onComplete(): Unit = { | ||
| val fetchInfos = fetchPartitionStatus.map { case (tp, status) => | ||
| val fetchInfos = fetchPartitionStatus.asScala.iterator.map { case (tp, status) => | ||
| tp -> status.fetchInfo | ||
| } | ||
| }.toBuffer | ||
|
|
||
| val logReadResults = replicaManager.readFromLog( | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Perhaps
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Iterators can be exhausted and hence are a bit more brittle. I would only use them as a method parameter if there is a meaningful difference in performance.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fair enough |
||
| params, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -76,6 +76,7 @@ import java.util.concurrent.atomic.AtomicBoolean | |
| import java.util.concurrent.{CompletableFuture, ConcurrentHashMap, Future, RejectedExecutionException, TimeUnit} | ||
| import java.util.{Collections, Optional, OptionalInt, OptionalLong} | ||
| import java.util.function.Consumer | ||
| import java.util.stream.Collectors | ||
| import scala.collection.{Map, Seq, Set, immutable, mutable} | ||
| import scala.jdk.CollectionConverters._ | ||
| import scala.jdk.OptionConverters.RichOptional | ||
|
|
@@ -1628,7 +1629,7 @@ class ReplicaManager(val config: KafkaConfig, | |
| params: FetchParams, | ||
| responseCallback: Seq[(TopicIdPartition, FetchPartitionData)] => Unit, | ||
| logReadResults: util.LinkedHashMap[TopicIdPartition, LogReadResult], | ||
| fetchPartitionStatus: Seq[(TopicIdPartition, FetchPartitionStatus)]): Unit = { | ||
| fetchPartitionStatus: util.LinkedHashMap[TopicIdPartition, FetchPartitionStatus]): Unit = { | ||
| val remoteFetchTasks = new util.HashMap[TopicIdPartition, Future[Void]] | ||
| val remoteFetchResults = new util.HashMap[TopicIdPartition, CompletableFuture[RemoteLogReadResult]] | ||
|
|
||
|
|
@@ -1643,7 +1644,7 @@ class ReplicaManager(val config: KafkaConfig, | |
| remoteFetchResults, | ||
| remoteFetchInfos, | ||
| remoteFetchMaxWaitMs, | ||
| fetchPartitionStatus.toMap.asJava, | ||
| fetchPartitionStatus, | ||
| params, | ||
| logReadResults, | ||
| tp => getPartitionOrException(tp), | ||
|
|
@@ -1710,17 +1711,17 @@ class ReplicaManager(val config: KafkaConfig, | |
| responseCallback(fetchPartitionData) | ||
| } else { | ||
| // construct the fetch results from the read results | ||
| val fetchPartitionStatus = new mutable.ArrayBuffer[(TopicIdPartition, FetchPartitionStatus)] | ||
| val fetchPartitionStatus = new util.LinkedHashMap[TopicIdPartition, FetchPartitionStatus] | ||
| fetchInfos.foreach { case (topicIdPartition, partitionData) => | ||
| val logReadResult = logReadResultMap.get(topicIdPartition) | ||
| if (logReadResult != null) { | ||
| val logOffsetMetadata = logReadResult.info.fetchOffsetMetadata | ||
| fetchPartitionStatus += (topicIdPartition -> new FetchPartitionStatus(logOffsetMetadata, partitionData)) | ||
| fetchPartitionStatus.put(topicIdPartition, new FetchPartitionStatus(logOffsetMetadata, partitionData)) | ||
| } | ||
| } | ||
|
|
||
| if (!remoteFetchInfos.isEmpty) { | ||
| processRemoteFetches(remoteFetchInfos, params, responseCallback, logReadResultMap, fetchPartitionStatus.toSeq) | ||
| processRemoteFetches(remoteFetchInfos, params, responseCallback, logReadResultMap, fetchPartitionStatus) | ||
| } else { | ||
| // If there is not enough data to respond and there is no remote data, we will let the fetch request | ||
| // wait for new data. | ||
|
|
@@ -1733,12 +1734,15 @@ class ReplicaManager(val config: KafkaConfig, | |
| ) | ||
|
|
||
| // create a list of (topic, partition) pairs to use as keys for this delayed fetch operation | ||
| val delayedFetchKeys = fetchPartitionStatus.map { case (tp, _) => new TopicPartitionOperationKey(tp) }.toList | ||
| val delayedFetchKeys = fetchPartitionStatus.keySet() | ||
| .stream() | ||
| .map(new TopicPartitionOperationKey(_)) | ||
| .collect(Collectors.toList[TopicPartitionOperationKey]()) | ||
|
||
|
|
||
| // try to complete the request immediately, otherwise put it into the purgatory; | ||
| // this is because while the delayed fetch operation is being created, new requests | ||
| // may arrive and hence make this operation completable. | ||
| delayedFetchPurgatory.tryCompleteElseWatch(delayedFetch, delayedFetchKeys.asJava) | ||
| delayedFetchPurgatory.tryCompleteElseWatch(delayedFetch, delayedFetchKeys) | ||
| } | ||
| } | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.