diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 268a06a79fd..510457ca46e 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -8,9 +8,6 @@ on: - develop - bazel -permissions: - actions: write - jobs: build: name: "bazel-compile (${{ matrix.os }})" @@ -23,13 +20,4 @@ jobs: - name: Build run: bazel build --config=remote //... - name: Run Tests - run: bazel test --config=remote //... - - name: Retry if failed - # if it failed , retry 2 times at most - if: failure() && fromJSON(github.run_attempt) < 3 - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "Attempting to retry workflow..." - gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }} \ No newline at end of file + run: bazel test --config=remote //... \ No newline at end of file diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 00000000000..8179f362879 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,45 @@ +name: Run Integration Tests +on: + pull_request: + types: [opened, reopened, synchronize] + push: + branches: [master, develop] + +jobs: + it-test: + name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + jdk: [8] + steps: + - name: Cache Maven Repos + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up JDK ${{ matrix.jdk }} + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.jdk }} + distribution: "adopt" + cache: "maven" + + - name: Run integration tests with Maven + run: mvn clean verify -Pit-test -Pskip-unit-tests + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() + with: + report_paths: 'test/target/failsafe-reports/TEST-*.xml' + annotate_only: true + include_passed: true + detailed_summary: true diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index 7d74c832be2..d0c0ba7d9f1 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -5,9 +5,6 @@ on: push: branches: [master, develop, bazel] -permissions: - actions: write - jobs: java_build: name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})" @@ -30,9 +27,6 @@ jobs: - name: Build with Maven run: mvn -B package --file pom.xml - - name: Run tests with increased memory and debug info - run: mvn test -X -Dparallel=none -DargLine="-Xmx1024m -XX:MaxPermSize=256m" - - name: Upload Auth JVM crash logs if: failure() uses: actions/upload-artifact@v4 @@ -41,26 +35,10 @@ jobs: path: /Users/runner/work/rocketmq/rocketmq/auth/hs_err_pid*.log retention-days: 1 - - name: Check for broker JVM crash logs - if: failure() - run: | - echo "Checking for JVM crash logs..." - ls -al /Users/runner/work/rocketmq/rocketmq/broker/ - - name: Upload broker JVM crash logs if: failure() uses: actions/upload-artifact@v4 with: name: jvm-crash-logs path: /Users/runner/work/rocketmq/rocketmq/broker/hs_err_pid*.log - retention-days: 1 - - - name: Retry if failed - # if it failed , retry 2 times at most - if: failure() && fromJSON(github.run_attempt) < 3 - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "Attempting to retry workflow..." - gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }} \ No newline at end of file + retention-days: 1 \ No newline at end of file diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index ef2db755d00..99d7309fd0c 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -21,7 +21,7 @@ jobs: - name: Build distribution tar run: | mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: rocketmq @@ -30,7 +30,7 @@ jobs: run: | mkdir -p ./pr echo ${{ github.event.number }} > ./pr/NR - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: pr path: pr/ diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml index f9bb3bde75a..5b4264266ef 100644 --- a/.github/workflows/pr-e2e-test.yml +++ b/.github/workflows/pr-e2e-test.yml @@ -25,18 +25,18 @@ jobs: java-version: ["8"] steps: - name: 'Download artifact' - uses: actions/github-script@v3.1.0 + uses: actions/github-script@v6 with: script: | - var artifacts = await github.actions.listWorkflowRunArtifacts({ + let artifacts = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, run_id: ${{github.event.workflow_run.id }}, }); - var matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => { + let matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => { return artifact.name == "rocketmq" })[0]; - var download = await github.actions.downloadArtifact({ + let download = await github.rest.actions.downloadArtifact({ owner: context.repo.owner, repo: context.repo.repo, artifact_id: matchArtifactRmq.id, @@ -68,7 +68,7 @@ jobs: mkdir versionlist touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`" sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: versionlist @@ -85,7 +85,7 @@ jobs: outputs: version-json: ${{ steps.show_versions.outputs.version-json }} steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download versionlist with: name: versionlist @@ -96,6 +96,7 @@ jobs: a=(`ls versionlist`) printf '%s\n' "${a[@]}" | jq -R . | jq -s . echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT + deploy: if: ${{ success() }} name: Deploy RocketMQ @@ -158,7 +159,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -199,7 +200,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -235,7 +236,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -258,5 +259,4 @@ jobs: action: "clean" ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" test-version: "${{ matrix.version }}" - job-id: ${{ strategy.job-index }} - + job-id: ${{ strategy.job-index }} \ No newline at end of file diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml index 2fe62dbeb06..b23d69788cb 100644 --- a/.github/workflows/push-ci.yml +++ b/.github/workflows/push-ci.yml @@ -31,7 +31,7 @@ jobs: - name: Build distribution tar run: | mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: rocketmq @@ -53,7 +53,7 @@ jobs: repository: apache/rocketmq-docker.git ref: master path: rocketmq-docker - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download distribution tar with: name: rocketmq @@ -72,7 +72,7 @@ jobs: mkdir versionlist touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`" sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: versionlist @@ -90,7 +90,7 @@ jobs: outputs: version-json: ${{ steps.show_versions.outputs.version-json }} steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download versionlist with: name: versionlist @@ -163,7 +163,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -204,7 +204,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -240,7 +240,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: diff --git a/.github/workflows/rerun-workflow.yml b/.github/workflows/rerun-workflow.yml index bf83fc51b63..6c319505d2c 100644 --- a/.github/workflows/rerun-workflow.yml +++ b/.github/workflows/rerun-workflow.yml @@ -1,21 +1,22 @@ name: Rerun workflow on: - workflow_dispatch: - inputs: - run_id: - required: true + workflow_run: + workflows: ["Build and Run Tests by Maven" , "Build and Run Tests by Bazel"] + types: + - completed permissions: actions: write jobs: rerun: + if: github.event.workflow_run.conclusion == 'failure' && fromJSON(github.event.workflow_run.run_attempt) < 3 runs-on: ubuntu-latest steps: - - name: rerun ${{ inputs.run_id }} + - name: rerun ${{ github.event.workflow_run.id }} env: - GH_REPO: ${{ github.repository }} + GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh run watch ${{ inputs.run_id }} > /dev/null 2>&1 - gh run rerun ${{ inputs.run_id }} --failed \ No newline at end of file + gh run watch ${{ github.event.workflow_run.id }} > /dev/null 2>&1 + gh run rerun ${{ github.event.workflow_run.id }} --failed \ No newline at end of file diff --git a/.github/workflows/snapshot-automation.yml b/.github/workflows/snapshot-automation.yml index 99855d3aa0d..9fb16cb13ca 100644 --- a/.github/workflows/snapshot-automation.yml +++ b/.github/workflows/snapshot-automation.yml @@ -69,7 +69,7 @@ jobs: MAVEN_SETTINGS: ${{ github.workspace }}/.github/asf-deploy-settings.xml run: | mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: rocketmq @@ -91,7 +91,7 @@ jobs: repository: apache/rocketmq-docker.git ref: master path: rocketmq-docker - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download distribution tar with: name: rocketmq @@ -110,7 +110,7 @@ jobs: mkdir versionlist touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`" sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: versionlist @@ -125,7 +125,7 @@ jobs: outputs: version-json: ${{ steps.show_versions.outputs.version-json }} steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download versionlist with: name: versionlist @@ -200,7 +200,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: diff --git a/acl/pom.xml b/acl/pom.xml index c9d5085dcc1..812dbd9fd13 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT rocketmq-acl rocketmq-acl ${project.version} diff --git a/auth/pom.xml b/auth/pom.xml index 71b07c33750..f7a5417860c 100644 --- a/auth/pom.xml +++ b/auth/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT rocketmq-auth rocketmq-auth ${project.version} diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java index 02d5df236f5..e69abdaf805 100644 --- a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java +++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java @@ -171,7 +171,7 @@ public List build(ChannelHandlerContext context, Re subject = User.of(fields.get(SessionCredentials.ACCESS_KEY)); } String remoteAddr = RemotingHelper.parseChannelRemoteAddr(context.channel()); - String sourceIp = StringUtils.substringBefore(remoteAddr, CommonConstants.COLON); + String sourceIp = StringUtils.substringBeforeLast(remoteAddr, CommonConstants.COLON); Resource topic; Resource group; @@ -394,7 +394,7 @@ private List newContext(Metadata metadata, QueryRou subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK)); } Resource resource = Resource.ofTopic(topic.getName()); - String sourceIp = StringUtils.substringBefore(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); + String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, Arrays.asList(Action.PUB, Action.SUB), sourceIp); return Collections.singletonList(context); } @@ -437,7 +437,7 @@ private static List newPubContext(Metadata metadata subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK)); } Resource resource = Resource.ofTopic(topic.getName()); - String sourceIp = StringUtils.substringBefore(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); + String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, Action.PUB, sourceIp); return Collections.singletonList(context); } @@ -483,7 +483,7 @@ private static List newSubContexts(Metadata metadat if (metadata.containsKey(GrpcConstants.AUTHORIZATION_AK)) { subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK)); } - String sourceIp = StringUtils.substringBefore(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); + String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); result.add(DefaultAuthorizationContext.of(subject, resource, Action.SUB, sourceIp)); return result; } diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java index f87a5304cb7..29748a9ed44 100644 --- a/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java +++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java @@ -105,7 +105,7 @@ public static AuthorizationEvaluator getEvaluator(AuthConfig config, Supplier public static AuthorizationStrategy getStrategy(AuthConfig config, Supplier metadataService) { try { Class clazz = StatelessAuthorizationStrategy.class; - if (StringUtils.isNotBlank(config.getAuthenticationStrategy())) { + if (StringUtils.isNotBlank(config.getAuthorizationStrategy())) { clazz = (Class) Class.forName(config.getAuthorizationStrategy()); } return clazz.getDeclaredConstructor(AuthConfig.class, Supplier.class).newInstance(config, metadataService); diff --git a/broker/pom.xml b/broker/pom.xml index 7f74059a969..f74c12989a1 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 145a9522306..aaf06caddf8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -18,7 +18,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import java.io.IOException; import java.net.InetSocketAddress; import java.util.AbstractMap; import java.util.ArrayList; @@ -747,7 +746,7 @@ public void run() { LOG.error("Failed to update nameServer address list", e); } } - }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); + }, 1000 * 10, this.brokerConfig.getUpdateNameServerAddrPeriod(), TimeUnit.MILLISECONDS); } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) { this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @@ -789,6 +788,9 @@ public boolean initializeMessageStore() { defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); } else { defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); + if (messageStoreConfig.isRocksdbCQDoubleWriteEnable()) { + defaultMessageStore.enableRocksdbCQWrite(); + } } if (messageStoreConfig.isEnableDLegerCommitLog()) { @@ -812,7 +814,7 @@ public boolean initializeMessageStore() { this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg)); this.messageStore.setTimerMessageStore(this.timerMessageStore); } - } catch (IOException e) { + } catch (Exception e) { result = false; LOG.error("BrokerController#initialize: unexpected error occurs", e); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java index 9f838b51544..b1057e2a8d4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java @@ -145,8 +145,9 @@ public boolean doChannelCloseEvent(final String remoteAddr, final Channel channe callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, next.getKey()); } } - - callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel()); + if (!isBroadcastMode(info.getMessageModel())) { + callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel()); + } } } return removed; @@ -196,7 +197,7 @@ public boolean registerConsumer(final String group, final ClientChannelInfo clie } if (r1 || r2) { - if (isNotifyConsumerIdsChangedEnable) { + if (isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) { callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); } } @@ -219,7 +220,7 @@ public boolean registerConsumerWithoutSub(final String group, final ClientChanne consumerGroupInfo = prev != null ? prev : tmp; } boolean updateChannelRst = consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, consumeFromWhere); - if (updateChannelRst && isNotifyConsumerIdsChangedEnable) { + if (updateChannelRst && isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) { callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); } if (null != this.brokerStatsManager) { @@ -244,7 +245,7 @@ public void unregisterConsumer(final String group, final ClientChannelInfo clien callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, group); } } - if (isNotifyConsumerIdsChangedEnable) { + if (isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) { callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); } } @@ -334,4 +335,8 @@ protected void callConsumerIdsChangeListener(ConsumerGroupEvent event, String gr } } } + + private boolean isBroadcastMode(final MessageModel messageModel) { + return MessageModel.BROADCASTING.equals(messageModel); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java index 21f20dde325..403324137cc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java @@ -31,6 +31,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerPathConfigHelper; import org.apache.rocketmq.common.ConfigManager; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; @@ -373,6 +374,25 @@ public void setDataVersion(DataVersion dataVersion) { this.dataVersion = dataVersion; } + public boolean loadDataVersion() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName); + if (jsonString != null) { + ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class); + if (obj != null) { + this.dataVersion = obj.dataVersion; + } + LOG.info("load consumer offset dataVersion success,{},{} ", fileName, jsonString); + } + return true; + } catch (Exception e) { + LOG.error("load consumer offset dataVersion failed " + fileName, e); + return false; + } + } + public void removeOffset(final String group) { Iterator>> it = this.offsetTable.entrySet().iterator(); while (it.hasNext()) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java index de293fc4992..1e7cda71eed 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java @@ -16,26 +16,31 @@ */ package org.apache.rocketmq.broker.offset; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; import java.io.File; import java.util.Iterator; import java.util.Map.Entry; import java.util.concurrent.ConcurrentMap; - import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.protocol.DataVersion; import org.rocksdb.WriteBatch; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; - public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + protected RocksDBConfigManager rocksDBConfigManager; public RocksDBConsumerOffsetManager(BrokerController brokerController) { super(brokerController); - this.rocksDBConfigManager = new RocksDBConfigManager(configFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override @@ -43,9 +48,47 @@ public boolean load() { if (!rocksDBConfigManager.init()) { return false; } - return this.rocksDBConfigManager.loadData(this::decodeOffset); + if (!loadDataVersion() || !loadConsumerOffset()) { + return false; + } + + return true; + } + + public boolean loadConsumerOffset() { + return this.rocksDBConfigManager.loadData(this::decodeOffset) && merge(); + } + + private boolean merge() { + if (!brokerController.getMessageStoreConfig().isTransferOffsetJsonToRocksdb()) { + log.info("the switch transferOffsetJsonToRocksdb is off, no merge offset operation is needed."); + return true; + } + if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { + log.info("consumerOffset json file does not exist, so skip merge"); + return true; + } + if (!super.loadDataVersion()) { + log.error("load json consumerOffset dataVersion error, startup will exit"); + return false; + } + + final DataVersion dataVersion = super.getDataVersion(); + final DataVersion kvDataVersion = this.getDataVersion(); + if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) { + if (!super.load()) { + log.error("load json consumerOffset info failed, startup will exit"); + return false; + } + this.persist(); + this.getDataVersion().assignNewOne(dataVersion); + updateDataVersion(); + log.info("update offset from json, dataVersion:{}, offsetTable: {} ", this.getDataVersion(), JSON.toJSONString(this.getOffsetTable())); + } + return true; } + @Override public boolean stop() { return this.rocksDBConfigManager.stop(); @@ -69,8 +112,7 @@ protected void decodeOffset(final byte[] key, final byte[] body) { LOG.info("load exist local offset, {}, {}", topicAtGroup, wrapper.getOffsetTable()); } - @Override - public String configFilePath() { + public String rocksdbConfigFilePath() { return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "consumerOffsets" + File.separator; } @@ -103,4 +145,23 @@ private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupN byte[] valueBytes = JSON.toJSONBytes(wrapper, SerializerFeature.BrowserCompatible); writeBatch.put(keyBytes, valueBytes); } + + @Override + public boolean loadDataVersion() { + return this.rocksDBConfigManager.loadDataVersion(); + } + + @Override + public DataVersion getDataVersion() { + return rocksDBConfigManager.getKvDataVersion(); + } + + public void updateDataVersion() { + try { + rocksDBConfigManager.updateKvDataVersion(); + } catch (Exception e) { + log.error("update consumer offset dataVersion error", e); + throw new RuntimeException(e); + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 3039cf5c97c..80f3f44facb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -18,9 +18,11 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import io.opentelemetry.api.common.Attributes; import java.io.UnsupportedEncodingException; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; @@ -38,7 +40,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import io.opentelemetry.api.common.Attributes; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.acl.AccessValidator; @@ -69,6 +70,7 @@ import org.apache.rocketmq.common.LockCallback; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UnlockCallback; @@ -137,6 +139,7 @@ import org.apache.rocketmq.remoting.protocol.body.TopicList; import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; import org.apache.rocketmq.remoting.protocol.body.UserInfo; +import org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CreateAccessConfigRequestHeader; @@ -209,16 +212,19 @@ import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.timer.TimerCheckpoint; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.LibC; -import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM; + import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_INVOCATION_STATUS; +import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM; import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; public class AdminBrokerProcessor implements NettyRequestProcessor { @@ -339,6 +345,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, return fetchAllConsumeStatsInBroker(ctx, request); case RequestCode.QUERY_CONSUME_QUEUE: return queryConsumeQueue(ctx, request); + case RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS: + return this.checkRocksdbCqWriteProgress(ctx, request); case RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN: return this.updateAndGetGroupForbidden(ctx, request); case RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG: @@ -458,6 +466,76 @@ private RemotingCommand updateAndGetGroupForbidden(ChannelHandlerContext ctx, Re return response; } + private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + CheckRocksdbCqWriteProgressRequestHeader requestHeader = request.decodeCommandCustomHeader(CheckRocksdbCqWriteProgressRequestHeader.class); + String requestTopic = requestHeader.getTopic(); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + MessageStore messageStore = brokerController.getMessageStore(); + DefaultMessageStore defaultMessageStore; + if (messageStore instanceof AbstractPluginMessageStore) { + defaultMessageStore = (DefaultMessageStore) ((AbstractPluginMessageStore) messageStore).getNext(); + } else { + defaultMessageStore = (DefaultMessageStore) messageStore; + } + RocksDBMessageStore rocksDBMessageStore = defaultMessageStore.getRocksDBMessageStore(); + if (!defaultMessageStore.getMessageStoreConfig().isRocksdbCQDoubleWriteEnable()) { + response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", "rocksdbCQWriteEnable is false, checkRocksdbCqWriteProgressCommand is invalid"))); + return response; + } + + ConcurrentMap> cqTable = defaultMessageStore.getConsumeQueueTable(); + StringBuilder diffResult = new StringBuilder(); + try { + if (StringUtils.isNotBlank(requestTopic)) { + processConsumeQueuesForTopic(cqTable.get(requestTopic), requestTopic, rocksDBMessageStore, diffResult,false); + response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", diffResult.toString()))); + return response; + } + for (Map.Entry> topicEntry : cqTable.entrySet()) { + String topic = topicEntry.getKey(); + processConsumeQueuesForTopic(topicEntry.getValue(), topic, rocksDBMessageStore, diffResult,true); + } + diffResult.append("check all topic successful, size:").append(cqTable.size()); + response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", diffResult.toString()))); + + } catch (Exception e) { + LOGGER.error("CheckRocksdbCqWriteProgressCommand error", e); + response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", e.getMessage()))); + } + return response; + } + + private void processConsumeQueuesForTopic(ConcurrentMap queueMap, String topic, RocksDBMessageStore rocksDBMessageStore, StringBuilder diffResult, boolean checkAll) { + for (Map.Entry queueEntry : queueMap.entrySet()) { + Integer queueId = queueEntry.getKey(); + ConsumeQueueInterface jsonCq = queueEntry.getValue(); + ConsumeQueueInterface kvCq = rocksDBMessageStore.getConsumeQueue(topic, queueId); + if (!checkAll) { + String format = String.format("\n[topic: %s, queue: %s] \n kvEarliest : %s | kvLatest : %s \n fileEarliest: %s | fileEarliest: %s ", + topic, queueId, kvCq.getEarliestUnit(), kvCq.getLatestUnit(), jsonCq.getEarliestUnit(), jsonCq.getLatestUnit()); + diffResult.append(format).append("\n"); + } + long maxFileOffsetInQueue = jsonCq.getMaxOffsetInQueue(); + long minOffsetInQueue = kvCq.getMinOffsetInQueue(); + for (long i = minOffsetInQueue; i < maxFileOffsetInQueue; i++) { + Pair fileCqUnit = jsonCq.getCqUnitAndStoreTime(i); + Pair kvCqUnit = kvCq.getCqUnitAndStoreTime(i); + if (fileCqUnit == null || kvCqUnit == null) { + diffResult.append(String.format("[topic: %s, queue: %s, offset: %s] \n kv : %s \n file : %s \n", + topic, queueId, i, kvCqUnit != null ? kvCqUnit.getObject1() : "null", fileCqUnit != null ? fileCqUnit.getObject1() : "null")); + return; + } + if (!checkCqUnitEqual(kvCqUnit.getObject1(), fileCqUnit.getObject1())) { + String diffInfo = String.format("[topic:%s, queue: %s offset: %s] \n file : %s \n kv : %s \n", + topic, queueId, i, kvCqUnit.getObject1(), fileCqUnit.getObject1()); + LOGGER.error(diffInfo); + diffResult.append(diffInfo).append(System.lineSeparator()); + return; + } + } + } + } @Override public boolean rejectRequest() { return false; @@ -2062,7 +2140,7 @@ private RemotingCommand resetOffsetInner(String topic, String group, int queueId Map queueOffsetMap = new HashMap<>(); // Reset offset for all queues belonging to the specified topic - TopicConfig topicConfig = brokerController.getTopicConfigManager().getTopicConfigTable().get(topic); + TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic); if (null == topicConfig) { response.setCode(ResponseCode.TOPIC_NOT_EXIST); response.setRemark("Topic " + topic + " does not exist"); @@ -3305,4 +3383,20 @@ private boolean validateBlackListConfigExist(Properties properties) { } return false; } + + private boolean checkCqUnitEqual(CqUnit cqUnit1, CqUnit cqUnit2) { + if (cqUnit1.getQueueOffset() != cqUnit2.getQueueOffset()) { + return false; + } + if (cqUnit1.getSize() != cqUnit2.getSize()) { + return false; + } + if (cqUnit1.getPos() != cqUnit2.getPos()) { + return false; + } + if (cqUnit1.getBatchNum() != cqUnit2.getBatchNum()) { + return false; + } + return cqUnit1.getTagsCode() == cqUnit2.getTagsCode(); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 47ef8e4013b..2d76c5a3caa 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -373,7 +373,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC // considered the same type because they share the same retry flag in previous fields. // Therefore, needRetryV1 is designed as a subset of needRetry, and within a single request, // only one type of retry topic is able to call popMsgFromQueue. - boolean needRetry = randomQ % 5 == 0; + boolean needRetry = randomQ < brokerConfig.getPopFromRetryProbability(); boolean needRetryV1 = false; if (brokerConfig.isEnableRetryTopicV2() && brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) { needRetryV1 = randomQ % 2 == 0; @@ -540,6 +540,7 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId, return future; } + future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey)); if (isPopShouldStop(topic, requestHeader.getConsumerGroup(), queueId)) { POP_LOGGER.warn("Too much msgs unacked, then stop poping. topic={}, group={}, queueId={}", topic, requestHeader.getConsumerGroup(), queueId); restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; @@ -548,7 +549,6 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId, } try { - future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey)); offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(), true, lockKey, true); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java index 018083811e8..69e59fd8e7f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java @@ -43,4 +43,13 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) } super.updateSubscriptionGroupConfig(config); } + + @Override + public boolean containsSubscriptionGroup(String group) { + if (MixAll.isLmq(group)) { + return true; + } else { + return super.containsSubscriptionGroup(group); + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java index 8c05d0bd98f..e4de25756bf 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java @@ -43,4 +43,13 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) } super.updateSubscriptionGroupConfig(config); } + + @Override + public boolean containsSubscriptionGroup(String group) { + if (MixAll.isLmq(group)) { + return true; + } else { + return super.containsSubscriptionGroup(group); + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java index 7df72dbe686..5119f78672c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java @@ -19,6 +19,12 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; +import java.io.File; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; import org.apache.rocketmq.common.UtilAll; @@ -27,13 +33,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.rocksdb.RocksIterator; -import java.io.File; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.BiConsumer; - public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { protected RocksDBConfigManager rocksDBConfigManager; @@ -79,28 +78,30 @@ public boolean loadForbidden(BiConsumer biConsumer) { private boolean merge() { if (!brokerController.getMessageStoreConfig().isTransferMetadataJsonToRocksdb()) { - log.info("The switch is off, no merge operation is needed."); + log.info("the switch transferMetadataJsonToRocksdb is off, no merge subGroup operation is needed."); return true; } if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { - log.info("json file and json back file not exist, so skip merge"); + log.info("subGroup json file does not exist, so skip merge"); return true; } - - if (!super.load()) { - log.error("load group and forbidden info from json file error, startup will exit"); + if (!super.loadDataVersion()) { + log.error("load json subGroup dataVersion error, startup will exit"); return false; } - - final ConcurrentMap groupTable = this.getSubscriptionGroupTable(); - final ConcurrentMap> forbiddenTable = this.getForbiddenTable(); final DataVersion dataVersion = super.getDataVersion(); final DataVersion kvDataVersion = this.getDataVersion(); if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) { + if (!super.load()) { + log.error("load group and forbidden info from json file error, startup will exit"); + return false; + } + final ConcurrentMap groupTable = this.getSubscriptionGroupTable(); for (Map.Entry entry : groupTable.entrySet()) { putSubscriptionGroupConfig(entry.getValue()); log.info("import subscription config to rocksdb, group={}", entry.getValue()); } + final ConcurrentMap> forbiddenTable = this.getForbiddenTable(); for (Map.Entry> entry : forbiddenTable.entrySet()) { try { this.rocksDBConfigManager.updateForbidden(entry.getKey(), JSON.toJSONString(entry.getValue())); @@ -110,8 +111,10 @@ private boolean merge() { return false; } } - this.rocksDBConfigManager.getKvDataVersion().assignNewOne(dataVersion); + this.getDataVersion().assignNewOne(dataVersion); updateDataVersion(); + } else { + log.info("dataVersion is not greater than kvDataVersion, no need to merge group metaData, dataVersion={}, kvDataVersion={}", dataVersion, kvDataVersion); } log.info("finish marge subscription config from json file and merge to rocksdb"); this.persist(); @@ -196,6 +199,7 @@ public void updateDataVersion() { try { rocksDBConfigManager.updateKvDataVersion(); } catch (Exception e) { + log.error("update group config dataVersion error", e); throw new RuntimeException(e); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java index f2a7e0482b1..e6855ef9a2a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -334,6 +334,26 @@ public DataVersion getDataVersion() { return dataVersion; } + public boolean loadDataVersion() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName); + if (jsonString != null) { + SubscriptionGroupManager obj = RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class); + if (obj != null) { + this.dataVersion.assignNewOne(obj.dataVersion); + this.printLoadDataWhenFirstBoot(obj); + log.info("load subGroup dataVersion success,{},{}", fileName, obj.dataVersion); + } + } + return true; + } catch (Exception e) { + log.error("load subGroup dataVersion failed" + fileName, e); + return false; + } + } + public void deleteSubscriptionGroupConfig(final String groupName) { SubscriptionGroupConfig old = removeSubscriptionGroupConfig(groupName); this.forbiddenTable.remove(groupName); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java index 2a89dd7e024..466e6416f98 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java @@ -18,6 +18,9 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; +import java.io.File; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; import org.apache.rocketmq.common.TopicConfig; @@ -25,10 +28,6 @@ import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.remoting.protocol.DataVersion; -import java.io.File; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; - public class RocksDBTopicConfigManager extends TopicConfigManager { protected RocksDBConfigManager rocksDBConfigManager; @@ -60,29 +59,35 @@ public boolean loadDataVersion() { private boolean merge() { if (!brokerController.getMessageStoreConfig().isTransferMetadataJsonToRocksdb()) { - log.info("The switch is off, no merge operation is needed."); + log.info("the switch transferMetadataJsonToRocksdb is off, no merge topic operation is needed."); return true; } if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { - log.info("json file and json back file not exist, so skip merge"); + log.info("topic json file does not exist, so skip merge"); return true; } - if (!super.load()) { - log.error("load topic config from json file error, startup will exit"); + if (!super.loadDataVersion()) { + log.error("load json topic dataVersion error, startup will exit"); return false; } - final ConcurrentMap topicConfigTable = this.getTopicConfigTable(); final DataVersion dataVersion = super.getDataVersion(); final DataVersion kvDataVersion = this.getDataVersion(); if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) { + if (!super.load()) { + log.error("load topic config from json file error, startup will exit"); + return false; + } + final ConcurrentMap topicConfigTable = this.getTopicConfigTable(); for (Map.Entry entry : topicConfigTable.entrySet()) { putTopicConfig(entry.getValue()); log.info("import topic config to rocksdb, topic={}", entry.getValue()); } - this.rocksDBConfigManager.getKvDataVersion().assignNewOne(dataVersion); + this.getDataVersion().assignNewOne(dataVersion); updateDataVersion(); + } else { + log.info("dataVersion is not greater than kvDataVersion, no need to merge topic metaData, dataVersion={}, kvDataVersion={}", dataVersion, kvDataVersion); } log.info("finish read topic config from json file and merge to rocksdb"); this.persist(); @@ -150,6 +155,7 @@ public void updateDataVersion() { try { rocksDBConfigManager.updateKvDataVersion(); } catch (Exception e) { + log.error("update topic config dataVersion error", e); throw new RuntimeException(e); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index eab2896b001..25d3218f2ab 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -637,6 +637,26 @@ public String encode() { return encode(false); } + public boolean loadDataVersion() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName); + if (jsonString != null) { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = + TopicConfigSerializeWrapper.fromJson(jsonString, TopicConfigSerializeWrapper.class); + if (topicConfigSerializeWrapper != null) { + this.dataVersion.assignNewOne(topicConfigSerializeWrapper.getDataVersion()); + log.info("load topic metadata dataVersion success {}, {}", fileName, topicConfigSerializeWrapper.getDataVersion()); + } + } + return true; + } catch (Exception e) { + log.error("load topic metadata dataVersion failed" + fileName, e); + return false; + } + } + @Override public String configFilePath() { return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java index 8c909824348..a23ad20037c 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java @@ -18,10 +18,7 @@ package org.apache.rocketmq.broker.client; import io.netty.channel.Channel; -import java.util.HashSet; -import java.util.Set; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.broker.client.net.Broker2Client; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; @@ -30,13 +27,25 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import java.util.HashSet; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -49,19 +58,10 @@ public class ConsumerManagerTest { private ConsumerManager consumerManager; - private DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener; - @Mock private BrokerController brokerController; - @Mock - private ConsumerFilterManager consumerFilterManager; - - private BrokerConfig brokerConfig = new BrokerConfig(); - - private Broker2Client broker2Client; - - private BrokerStatsManager brokerStatsManager; + private final BrokerConfig brokerConfig = new BrokerConfig(); private static final String GROUP = "DEFAULT_GROUP"; @@ -74,40 +74,38 @@ public class ConsumerManagerTest { @Before public void before() { clientChannelInfo = new ClientChannelInfo(channel, CLIENT_ID, LanguageCode.JAVA, VERSION); - defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController); - brokerStatsManager = new BrokerStatsManager(brokerConfig); - consumerManager = new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig); - broker2Client = new Broker2Client(brokerController); + DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController); + BrokerStatsManager brokerStatsManager = new BrokerStatsManager(brokerConfig); + consumerManager = spy(new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig)); + ConsumerFilterManager consumerFilterManager = mock(ConsumerFilterManager.class); when(brokerController.getConsumerFilterManager()).thenReturn(consumerFilterManager); - when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); - when(brokerController.getBroker2Client()).thenReturn(broker2Client); } @Test public void compensateBasicConsumerInfoTest() { ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true); - Assertions.assertThat(consumerGroupInfo).isNull(); + assertThat(consumerGroupInfo).isNull(); consumerManager.compensateBasicConsumerInfo(GROUP, ConsumeType.CONSUME_ACTIVELY, MessageModel.BROADCASTING); consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true); - Assertions.assertThat(consumerGroupInfo).isNotNull(); - Assertions.assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY); - Assertions.assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING); + assertThat(consumerGroupInfo).isNotNull(); + assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY); + assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING); } @Test public void compensateSubscribeDataTest() { ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true); - Assertions.assertThat(consumerGroupInfo).isNull(); + assertThat(consumerGroupInfo).isNull(); consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL)); consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true); - Assertions.assertThat(consumerGroupInfo).isNotNull(); - Assertions.assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1); + assertThat(consumerGroupInfo).isNotNull(); + assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1); SubscriptionData subscriptionData = consumerGroupInfo.getSubscriptionTable().get(TOPIC); - Assertions.assertThat(subscriptionData).isNotNull(); - Assertions.assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC); - Assertions.assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL); + assertThat(subscriptionData).isNotNull(); + assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC); + assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL); } @Test @@ -118,7 +116,8 @@ public void registerConsumerTest() { subList.add(subscriptionData); consumerManager.registerConsumer(GROUP, clientChannelInfo, ConsumeType.CONSUME_PASSIVELY, MessageModel.BROADCASTING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET, subList, true); - Assertions.assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull(); + verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any()); + assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull(); } @Test @@ -128,63 +127,65 @@ public void unregisterConsumerTest() { // unregister consumerManager.unregisterConsumer(GROUP, clientChannelInfo, true); - Assertions.assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull(); + verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any()); + assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull(); } @Test public void findChannelTest() { register(); final ClientChannelInfo consumerManagerChannel = consumerManager.findChannel(GROUP, CLIENT_ID); - Assertions.assertThat(consumerManagerChannel).isNotNull(); + assertThat(consumerManagerChannel).isNotNull(); } @Test public void findSubscriptionDataTest() { register(); final SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC); - Assertions.assertThat(subscriptionData).isNotNull(); + assertThat(subscriptionData).isNotNull(); } @Test public void findSubscriptionDataCountTest() { register(); final int count = consumerManager.findSubscriptionDataCount(GROUP); - assert count > 0; + assertTrue(count > 0); } @Test public void findSubscriptionTest() { SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true); - Assertions.assertThat(subscriptionData).isNull(); + assertThat(subscriptionData).isNull(); consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL)); subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true); - Assertions.assertThat(subscriptionData).isNotNull(); - Assertions.assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC); - Assertions.assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL); + assertThat(subscriptionData).isNotNull(); + assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC); + assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL); subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, false); - Assertions.assertThat(subscriptionData).isNull(); + assertThat(subscriptionData).isNull(); } @Test public void scanNotActiveChannelTest() { clientChannelInfo.setLastUpdateTimestamp(System.currentTimeMillis() - brokerConfig.getChannelExpiredTimeout() * 2); consumerManager.scanNotActiveChannel(); - Assertions.assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0); + assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0); } @Test public void queryTopicConsumeByWhoTest() { register(); final HashSet consumeGroup = consumerManager.queryTopicConsumeByWho(TOPIC); - assert consumeGroup.size() > 0; + assertFalse(consumeGroup.isEmpty()); } @Test public void doChannelCloseEventTest() { consumerManager.doChannelCloseEvent("127.0.0.1", channel); - assert consumerManager.getConsumerTable().size() == 0; + verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any()); + assertEquals(0, consumerManager.getConsumerTable().size()); } private void register() { @@ -203,8 +204,8 @@ public void removeExpireConsumerGroupInfo() { consumerManager.compensateSubscribeData(GROUP, TOPIC, subscriptionData); consumerManager.compensateSubscribeData(GROUP, TOPIC + "_1", new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL)); consumerManager.removeExpireConsumerGroupInfo(); - Assertions.assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull(); - Assertions.assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull(); - Assertions.assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + "_1")).isNotNull(); + assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull(); + assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull(); + assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + "_1")).isNotNull(); } } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java new file mode 100644 index 00000000000..865e7b608ea --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.client.net; + +import io.netty.channel.Channel; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; +import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MQVersion; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.RemotingServer; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody; +import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.store.MessageStore; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class Broker2ClientTest { + + @Mock + private BrokerController brokerController; + + @Mock + private RemotingServer remotingServer; + + @Mock + private ConsumerManager consumerManager; + + @Mock + private TopicConfigManager topicConfigManager; + + @Mock + private ConsumerOffsetManager consumerOffsetManager; + + @Mock + private Channel channel; + + @Mock + private ConsumerGroupInfo consumerGroupInfo; + + private Broker2Client broker2Client; + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultGroup = "defaultGroup"; + + private final long timestamp = System.currentTimeMillis(); + + private final boolean isForce = true; + + @Before + public void init() { + broker2Client = new Broker2Client(brokerController); + when(brokerController.getRemotingServer()).thenReturn(remotingServer); + when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); + when(brokerController.getConsumerManager()).thenReturn(consumerManager); + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + when(brokerController.getBrokerConfig()).thenReturn(mock(BrokerConfig.class)); + when(brokerController.getMessageStore()).thenReturn(mock(MessageStore.class)); + when(consumerManager.getConsumerGroupInfo(any())).thenReturn(consumerGroupInfo); + } + + @Test + public void testCheckProducerTransactionState() throws Exception { + CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader(); + broker2Client.checkProducerTransactionState("group", channel, requestHeader, createMessageExt()); + verify(remotingServer).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L)); + } + + @Test + public void testCheckProducerTransactionStateException() throws Exception { + CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader(); + MessageExt messageExt = createMessageExt(); + doThrow(new RuntimeException("Test Exception")) + .when(remotingServer) + .invokeOneway(any(Channel.class), + any(RemotingCommand.class), + anyLong()); + broker2Client.checkProducerTransactionState("group", channel, requestHeader, messageExt); + verify(brokerController.getRemotingServer()).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L)); + } + + @Test + public void testResetOffsetNoTopicConfig() { + when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null); + RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); + assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + } + + @Test + public void testResetOffsetNoConsumerGroupInfo() { + TopicConfig topicConfig = mock(TopicConfig.class); + when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig); + when(topicConfig.getWriteQueueNums()).thenReturn(1); + when(consumerOffsetManager.queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L); + RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); + assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode()); + } + + @Test + public void testResetOffset() { + TopicConfig topicConfig = mock(TopicConfig.class); + when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig); + when(topicConfig.getWriteQueueNums()).thenReturn(1); + when(brokerController.getConsumerOffsetManager().queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L); + BrokerConfig brokerConfig = mock(BrokerConfig.class); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + when(brokerConfig.getBrokerName()).thenReturn(defaultBroker); + ConsumerGroupInfo consumerGroupInfo = mock(ConsumerGroupInfo.class); + when(consumerManager.getConsumerGroupInfo(defaultGroup)).thenReturn(consumerGroupInfo); + RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); + assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode()); + } + + @Test + public void testGetConsumeStatusNoConsumerOnline() { + when(consumerGroupInfo.getChannelInfoTable()).thenReturn(new ConcurrentHashMap<>()); + RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, ""); + assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + } + + @Test + public void testGetConsumeStatusClientDoesNotSupportFeature() { + ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, "defaultClientId", null, MQVersion.Version.V3_0_6.ordinal()); + ConcurrentMap channelInfoTable = new ConcurrentHashMap<>(); + channelInfoTable.put(channel, clientChannelInfo); + when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable); + RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, ""); + assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + } + + @Test + public void testGetConsumeStatus() throws Exception { + ConcurrentMap channelInfoTable = new ConcurrentHashMap<>(); + ClientChannelInfo clientChannelInfo = mock(ClientChannelInfo.class); + when(clientChannelInfo.getVersion()).thenReturn(MQVersion.CURRENT_VERSION); + channelInfoTable.put(channel, clientChannelInfo); + when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable); + RemotingCommand responseMock = mock(RemotingCommand.class); + when(responseMock.getCode()).thenReturn(ResponseCode.SUCCESS); + when(responseMock.getBody()).thenReturn("{\"consumerTable\":{}}".getBytes(StandardCharsets.UTF_8)); + when(remotingServer.invokeSync(any(Channel.class), any(RemotingCommand.class), anyLong())).thenReturn(responseMock); + RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, ""); + assertEquals(ResponseCode.SUCCESS, response.getCode()); + GetConsumerStatusBody body = RemotingSerializable.decode(response.getBody(), GetConsumerStatusBody.class); + assertEquals(1, body.getConsumerTable().size()); + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + result.setKeys("keys"); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setStoreHost(storeHost); + result.setBornHost(bornHost); + return result; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java new file mode 100644 index 00000000000..e231d61b6a7 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.client.rebalance; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.common.message.MessageQueue; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RebalanceLockManagerTest { + + @Mock + private RebalanceLockManager.LockEntry lockEntry; + + private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager(); + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultGroup = "defaultGroup"; + + private final String defaultClientId = "defaultClientId"; + + @Test + public void testIsLockAllExpiredGroupNotExist() { + assertTrue(rebalanceLockManager.isLockAllExpired(defaultGroup)); + } + + @Test + public void testIsLockAllExpiredGroupExist() throws IllegalAccessException { + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + when(lockEntry.isExpired()).thenReturn(false); + assertFalse(rebalanceLockManager.isLockAllExpired(defaultGroup)); + } + + @Test + public void testIsLockAllExpiredGroupExistSomeExpired() throws IllegalAccessException { + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + when(lockEntry.isExpired()).thenReturn(true).thenReturn(false); + assertFalse(rebalanceLockManager.isLockAllExpired(defaultGroup)); + } + + @Test + public void testTryLockNotLocked() { + assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId)); + } + + @Test + public void testTryLockSameClient() throws IllegalAccessException { + when(lockEntry.isLocked(defaultClientId)).thenReturn(true); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId)); + } + + @Test + public void testTryLockDifferentClient() throws Exception { + when(lockEntry.isLocked(defaultClientId)).thenReturn(false); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + assertFalse(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId)); + } + + @Test + public void testTryLockButExpired() throws IllegalAccessException { + when(lockEntry.isExpired()).thenReturn(true); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId)); + } + + @Test + public void testTryLockBatchAllLocked() { + Set mqs = createMessageQueue(2); + Set actual = rebalanceLockManager.tryLockBatch(defaultGroup, mqs, defaultClientId); + assertEquals(mqs, actual); + } + + @Test + public void testTryLockBatchNoneLocked() throws IllegalAccessException { + when(lockEntry.isLocked(defaultClientId)).thenReturn(false); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + Set actual = rebalanceLockManager.tryLockBatch(defaultGroup, createMessageQueue(2), defaultClientId); + assertTrue(actual.isEmpty()); + } + + @Test + public void testTryLockBatchSomeLocked() throws IllegalAccessException { + Set mqs = new HashSet<>(); + MessageQueue mq1 = new MessageQueue(defaultTopic, defaultBroker, 0); + MessageQueue mq2 = new MessageQueue(defaultTopic, defaultBroker, 1); + mqs.add(mq1); + mqs.add(mq2); + when(lockEntry.isLocked(defaultClientId)).thenReturn(true).thenReturn(false); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + Set actual = rebalanceLockManager.tryLockBatch(defaultGroup, mqs, defaultClientId); + Set expected = new HashSet<>(); + expected.add(mq2); + assertEquals(expected, actual); + } + + @Test + public void testUnlockBatch() throws IllegalAccessException { + when(lockEntry.getClientId()).thenReturn(defaultClientId); + ConcurrentMap> mqLockTable = createMQLockTable(); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", mqLockTable, true); + rebalanceLockManager.unlockBatch(defaultGroup, createMessageQueue(1), defaultClientId); + assertEquals(1, mqLockTable.get(defaultGroup).values().size()); + } + + @Test + public void testUnlockBatchByOtherClient() throws IllegalAccessException { + when(lockEntry.getClientId()).thenReturn("otherClientId"); + ConcurrentMap> mqLockTable = createMQLockTable(); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", mqLockTable, true); + rebalanceLockManager.unlockBatch(defaultGroup, createMessageQueue(1), defaultClientId); + assertEquals(2, mqLockTable.get(defaultGroup).values().size()); + } + + private MessageQueue createDefaultMessageQueue() { + return createMessageQueue(1).iterator().next(); + } + + private Set createMessageQueue(final int count) { + Set result = new HashSet<>(); + for (int i = 0; i < count; i++) { + result.add(new MessageQueue(defaultTopic, defaultBroker, i)); + } + return result; + } + + private ConcurrentMap> createMQLockTable() { + MessageQueue messageQueue1 = new MessageQueue(defaultTopic, defaultBroker, 0); + MessageQueue messageQueue2 = new MessageQueue(defaultTopic, defaultBroker, 1); + ConcurrentHashMap lockEntryMap = new ConcurrentHashMap<>(); + lockEntryMap.put(messageQueue1, lockEntry); + lockEntryMap.put(messageQueue2, lockEntry); + ConcurrentMap> result = new ConcurrentHashMap<>(); + result.put(defaultGroup, lockEntryMap); + return result; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java index d01a6f76f5e..39ec0d8d94f 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.broker.controller; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.slave.SlaveSynchronize; @@ -36,29 +37,31 @@ import org.apache.rocketmq.store.ha.autoswitch.BrokerMetadata; import org.apache.rocketmq.store.ha.autoswitch.TempBrokerMetadata; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; +import org.mockito.junit.MockitoJUnitRunner; import java.io.File; import java.time.Duration; -import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.UUID; import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; -@RunWith(PowerMockRunner.class) -@PrepareForTest(ReplicasManager.class) +@RunWith(MockitoJUnitRunner.class) public class ReplicasManagerRegisterTest { public static final String STORE_BASE_PATH = System.getProperty("java.io.tmpdir") + File.separator + "ReplicasManagerRegisterTest"; @@ -74,7 +77,14 @@ public class ReplicasManagerRegisterTest { public static final String CONTROLLER_ADDR = "127.0.0.1:8888"; public static final BrokerConfig BROKER_CONFIG; - private final HashSet syncStateSet = new HashSet<>(Arrays.asList(1L)); + + private final HashSet syncStateSet = new HashSet<>(Collections.singletonList(1L)); + + @Mock + private BrokerMetadata brokerMetadata; + + @Mock + private TempBrokerMetadata tempBrokerMetadata; static { BROKER_CONFIG = new BrokerConfig(); @@ -133,18 +143,19 @@ public void testBrokerRegisterSuccess() throws Exception { when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L)); when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader()); when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); - when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); + when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())) + .thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); ReplicasManager replicasManager0 = new ReplicasManager(mockedBrokerController); replicasManager0.start(); await().atMost(Duration.ofMillis(1000)).until(() -> replicasManager0.getState() == ReplicasManager.State.RUNNING ); - Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState()); - Assert.assertEquals(1L, replicasManager0.getBrokerControllerId().longValue()); + assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState()); + assertEquals(1L, replicasManager0.getBrokerControllerId().longValue()); checkMetadataFile(replicasManager0.getBrokerMetadata(), 1L); - Assert.assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded()); - Assert.assertFalse(replicasManager0.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded()); + assertFalse(replicasManager0.getTempBrokerMetadata().fileExists()); replicasManager0.shutdown(); } @@ -160,18 +171,18 @@ public void testBrokerRegisterSuccessAndRestartWithChangedBrokerConfig() throws await().atMost(Duration.ofMillis(1000)).until(() -> replicasManager0.getState() == ReplicasManager.State.RUNNING ); - Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState()); - Assert.assertEquals(1L, replicasManager0.getBrokerControllerId().longValue()); + assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState()); + assertEquals(1L, replicasManager0.getBrokerControllerId().longValue()); checkMetadataFile(replicasManager0.getBrokerMetadata(), 1L); - Assert.assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded()); - Assert.assertFalse(replicasManager0.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded()); + assertFalse(replicasManager0.getTempBrokerMetadata().fileExists()); replicasManager0.shutdown(); // change broker name in broker config mockedBrokerController.getBrokerConfig().setBrokerName(BROKER_NAME + "1"); ReplicasManager replicasManagerRestart = new ReplicasManager(mockedBrokerController); replicasManagerRestart.start(); - Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState()); + assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState()); mockedBrokerController.getBrokerConfig().setBrokerName(BROKER_NAME); replicasManagerRestart.shutdown(); @@ -179,7 +190,7 @@ public void testBrokerRegisterSuccessAndRestartWithChangedBrokerConfig() throws mockedBrokerController.getBrokerConfig().setBrokerClusterName(CLUSTER_NAME + "1"); replicasManagerRestart = new ReplicasManager(mockedBrokerController); replicasManagerRestart.start(); - Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState()); + assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState()); mockedBrokerController.getBrokerConfig().setBrokerClusterName(CLUSTER_NAME); replicasManagerRestart.shutdown(); } @@ -190,32 +201,29 @@ public void testRegisterFailedAtGetNextBrokerId() throws Exception { when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenThrow(new RuntimeException()); replicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.INITIAL, replicasManager.getRegisterState()); - Assert.assertFalse(replicasManager.getTempBrokerMetadata().fileExists()); - Assert.assertFalse(replicasManager.getBrokerMetadata().fileExists()); - Assert.assertNull(replicasManager.getBrokerControllerId()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); + assertEquals(ReplicasManager.RegisterState.INITIAL, replicasManager.getRegisterState()); + assertFalse(replicasManager.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManager.getBrokerMetadata().fileExists()); + assertNull(replicasManager.getBrokerControllerId()); replicasManager.shutdown(); } @Test public void testRegisterFailedAtCreateTempFile() throws Exception { - ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController); + ReplicasManager spyReplicasManager = new ReplicasManager(mockedBrokerController); when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L)); - when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader()); - when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); - when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); - ReplicasManager spyReplicasManager = PowerMockito.spy(replicasManager); - PowerMockito.doReturn(false).when(spyReplicasManager, "createTempMetadataFile", anyLong()); + FieldUtils.writeDeclaredField(spyReplicasManager, "tempBrokerMetadata", tempBrokerMetadata, true); + doThrow(new RuntimeException("Test exception")).when(tempBrokerMetadata).updateAndPersist(any(), any(), anyLong(), any()); spyReplicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.INITIAL, spyReplicasManager.getRegisterState()); - Assert.assertFalse(spyReplicasManager.getTempBrokerMetadata().fileExists()); - Assert.assertFalse(spyReplicasManager.getBrokerMetadata().fileExists()); - Assert.assertNull(spyReplicasManager.getBrokerControllerId()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState()); + assertEquals(ReplicasManager.RegisterState.INITIAL, spyReplicasManager.getRegisterState()); + assertFalse(spyReplicasManager.getTempBrokerMetadata().fileExists()); + assertFalse(spyReplicasManager.getBrokerMetadata().fileExists()); + assertNull(spyReplicasManager.getBrokerControllerId()); spyReplicasManager.shutdown(); } @@ -224,61 +232,57 @@ public void testRegisterFailedAtApplyBrokerIdFailed() throws Exception { ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController); when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L)); when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenThrow(new RuntimeException()); - when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); - when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); replicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); - Assert.assertNotEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState()); - Assert.assertNotEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager.getRegisterState()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); + assertNotEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState()); + assertNotEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager.getRegisterState()); replicasManager.shutdown(); - - Assert.assertFalse(replicasManager.getBrokerMetadata().fileExists()); - Assert.assertNull(replicasManager.getBrokerControllerId()); + + assertFalse(replicasManager.getBrokerMetadata().fileExists()); + assertNull(replicasManager.getBrokerControllerId()); } @Test public void testRegisterFailedAtCreateMetadataFileAndDeleteTemp() throws Exception { - ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController); + ReplicasManager spyReplicasManager = new ReplicasManager(mockedBrokerController); when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L)); when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader()); when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); - - ReplicasManager spyReplicasManager = PowerMockito.spy(replicasManager); - PowerMockito.doReturn(false).when(spyReplicasManager, "createMetadataFileAndDeleteTemp"); + + FieldUtils.writeDeclaredField(spyReplicasManager, "brokerMetadata", brokerMetadata, true); + doThrow(new RuntimeException("Test exception")).when(brokerMetadata).updateAndPersist(any(), any(), anyLong()); spyReplicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.CREATE_TEMP_METADATA_FILE_DONE, spyReplicasManager.getRegisterState()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState()); + assertEquals(ReplicasManager.RegisterState.CREATE_TEMP_METADATA_FILE_DONE, spyReplicasManager.getRegisterState()); TempBrokerMetadata tempBrokerMetadata = spyReplicasManager.getTempBrokerMetadata(); - Assert.assertTrue(tempBrokerMetadata.fileExists()); - Assert.assertTrue(tempBrokerMetadata.isLoaded()); - Assert.assertFalse(spyReplicasManager.getBrokerMetadata().fileExists()); - Assert.assertNull(spyReplicasManager.getBrokerControllerId()); + assertTrue(tempBrokerMetadata.fileExists()); + assertTrue(tempBrokerMetadata.isLoaded()); + assertFalse(spyReplicasManager.getBrokerMetadata().fileExists()); + assertNull(spyReplicasManager.getBrokerControllerId()); spyReplicasManager.shutdown(); // restart, we expect that this replicasManager still keep the tempMetadata and still try to finish its registering ReplicasManager replicasManagerNew = new ReplicasManager(mockedBrokerController); - // because apply brokerId: 1 has succeeded, so now next broker id is 2 - when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 2L)); replicasManagerNew.start(); - - Assert.assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState()); + + assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState()); + assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState()); // tempMetadata has been cleared - Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists()); - Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded()); + assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded()); // metadata has been persisted - Assert.assertTrue(replicasManagerNew.getBrokerMetadata().fileExists()); - Assert.assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded()); - Assert.assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue()); - Assert.assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue()); + assertTrue(replicasManagerNew.getBrokerMetadata().fileExists()); + assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded()); + assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue()); + assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue()); replicasManagerNew.shutdown(); } @@ -291,62 +295,57 @@ public void testRegisterFailedAtRegisterSuccess() throws Exception { when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); replicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); + assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState()); TempBrokerMetadata tempBrokerMetadata = replicasManager.getTempBrokerMetadata(); // temp metadata has been cleared - Assert.assertFalse(tempBrokerMetadata.fileExists()); - Assert.assertFalse(tempBrokerMetadata.isLoaded()); + assertFalse(tempBrokerMetadata.fileExists()); + assertFalse(tempBrokerMetadata.isLoaded()); // metadata has been persisted - Assert.assertTrue(replicasManager.getBrokerMetadata().fileExists()); - Assert.assertTrue(replicasManager.getBrokerMetadata().isLoaded()); - Assert.assertEquals(1L, replicasManager.getBrokerMetadata().getBrokerId().longValue()); - Assert.assertEquals(1L, replicasManager.getBrokerControllerId().longValue()); + assertTrue(replicasManager.getBrokerMetadata().fileExists()); + assertTrue(replicasManager.getBrokerMetadata().isLoaded()); + assertEquals(1L, replicasManager.getBrokerMetadata().getBrokerId().longValue()); + assertEquals(1L, replicasManager.getBrokerControllerId().longValue()); replicasManager.shutdown(); Mockito.reset(mockedBrokerOuterAPI); - when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); + when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())) + .thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); when(mockedBrokerOuterAPI.getControllerMetaData(any())).thenReturn( new GetMetaDataResponseHeader("default-group", "dledger-a", CONTROLLER_ADDR, true, CONTROLLER_ADDR)); when(mockedBrokerOuterAPI.checkAddressReachable(any())).thenReturn(true); // restart, we expect that this replicasManager still keep the metadata and still try to finish its registering ReplicasManager replicasManagerNew = new ReplicasManager(mockedBrokerController); - // because apply brokerId: 1 has succeeded, so now next broker id is 2 - when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 2L)); - // because apply brokerId: 1 has succeeded, so next request which try to apply brokerId: 1 will be failed - when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), eq(1L), any(), any())).thenThrow(new RuntimeException()); when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); replicasManagerNew.start(); - - Assert.assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState()); + + assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState()); + assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState()); // tempMetadata has been cleared - Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists()); - Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded()); + assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded()); // metadata has been persisted - Assert.assertTrue(replicasManagerNew.getBrokerMetadata().fileExists()); - Assert.assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded()); - Assert.assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue()); - Assert.assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue()); + assertTrue(replicasManagerNew.getBrokerMetadata().fileExists()); + assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded()); + assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue()); + assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue()); replicasManagerNew.shutdown(); } private void checkMetadataFile(BrokerMetadata brokerMetadata0 ,Long brokerId) throws Exception { - Assert.assertEquals(brokerId, brokerMetadata0.getBrokerId()); - Assert.assertTrue(brokerMetadata0.fileExists()); + assertEquals(brokerId, brokerMetadata0.getBrokerId()); + assertTrue(brokerMetadata0.fileExists()); BrokerMetadata brokerMetadata = new BrokerMetadata(brokerMetadata0.getFilePath()); brokerMetadata.readFromFile(); - Assert.assertEquals(brokerMetadata0, brokerMetadata); + assertEquals(brokerMetadata0, brokerMetadata); } @After public void clear() { UtilAll.deleteFile(new File(STORE_BASE_PATH)); } - - } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java new file mode 100644 index 00000000000..b4800aec24e --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.offset; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.commons.collections.MapUtils; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.RocksDBMessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; +import org.apache.rocketmq.store.queue.CqUnit; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.rocksdb.RocksDBException; + +@RunWith(MockitoJUnitRunner.class) +public class RocksdbTransferOffsetAndCqTest { + + private final String basePath = Paths.get(System.getProperty("user.home"), + "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString(); + + private final String topic = "topic"; + private final String group = "group"; + private final String clientHost = "clientHost"; + private final int queueId = 1; + + private RocksDBConsumerOffsetManager rocksdbConsumerOffsetManager; + + private ConsumerOffsetManager consumerOffsetManager; + + private DefaultMessageStore defaultMessageStore; + + @Mock + private BrokerController brokerController; + + @Before + public void init() throws IOException { + if (notToBeExecuted()) { + return; + } + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setConsumerOffsetUpdateVersionStep(10); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(basePath); + messageStoreConfig.setTransferOffsetJsonToRocksdb(true); + messageStoreConfig.setRocksdbCQDoubleWriteEnable(true); + Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + + defaultMessageStore = new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("aaa", true), null, + brokerConfig, new ConcurrentHashMap()); + defaultMessageStore.enableRocksdbCQWrite(); + defaultMessageStore.loadCheckPoint(); + + consumerOffsetManager = new ConsumerOffsetManager(brokerController); + consumerOffsetManager.load(); + + rocksdbConsumerOffsetManager = new RocksDBConsumerOffsetManager(brokerController); + } + + @Test + public void testTransferOffset() { + if (notToBeExecuted()) { + return; + } + + for (int i = 0; i < 200; i++) { + consumerOffsetManager.commitOffset(clientHost, group, topic, queueId, i); + } + + ConcurrentMap> offsetTable = consumerOffsetManager.getOffsetTable(); + ConcurrentMap map = offsetTable.get(topic + "@" + group); + Assert.assertTrue(MapUtils.isNotEmpty(map)); + + Long offset = map.get(queueId); + Assert.assertEquals(199L, (long) offset); + + long offsetDataVersion = consumerOffsetManager.getDataVersion().getCounter().get(); + Assert.assertEquals(20L, offsetDataVersion); + + consumerOffsetManager.persist(); + + boolean loadResult = rocksdbConsumerOffsetManager.load(); + Assert.assertTrue(loadResult); + + ConcurrentMap> rocksdbOffsetTable = rocksdbConsumerOffsetManager.getOffsetTable(); + + ConcurrentMap rocksdbMap = rocksdbOffsetTable.get(topic + "@" + group); + Assert.assertTrue(MapUtils.isNotEmpty(rocksdbMap)); + + Long aLong1 = rocksdbMap.get(queueId); + Assert.assertEquals(199L, (long) aLong1); + + long rocksdbOffset = rocksdbConsumerOffsetManager.getDataVersion().getCounter().get(); + Assert.assertEquals(21L, rocksdbOffset); + } + + @Test + public void testRocksdbCqWrite() throws RocksDBException { + if (notToBeExecuted()) { + return; + } + RocksDBMessageStore kvStore = defaultMessageStore.getRocksDBMessageStore(); + ConsumeQueueStoreInterface store = kvStore.getConsumeQueueStore(); + ConsumeQueueInterface rocksdbCq = defaultMessageStore.getRocksDBMessageStore().findConsumeQueue(topic, queueId); + ConsumeQueueInterface fileCq = defaultMessageStore.findConsumeQueue(topic, queueId); + for (int i = 0; i < 200; i++) { + DispatchRequest request = new DispatchRequest(topic, queueId, i, 200, 0, System.currentTimeMillis(), i, "", "", 0, 0, new HashMap<>()); + fileCq.putMessagePositionInfoWrapper(request); + store.putMessagePositionInfoWrapper(request); + } + Pair unit = rocksdbCq.getCqUnitAndStoreTime(100); + Pair unit1 = fileCq.getCqUnitAndStoreTime(100); + Assert.assertTrue(unit.getObject1().getPos() == unit1.getObject1().getPos()); + } + + private boolean notToBeExecuted() { + return MixAll.isMac(); + } + +} diff --git a/client/pom.xml b/client/pom.xml index 5a6c92f97dd..e13d106a17d 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index 089fd39b3e9..7c9a65ecdbf 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -262,7 +262,7 @@ public void setRegisterTopics(Set registerTopics) { public void sendMessageBack(MessageExt msg, int delayLevel) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { msg.setTopic(withNamespace(msg.getTopic())); - this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, null); + this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, msg.getBrokerName()); } /** diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 94785c69708..5df5cc8fa1a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -688,7 +688,7 @@ public void setSubscription(Map subscription) { public void sendMessageBack(MessageExt msg, int delayLevel) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { msg.setTopic(withNamespace(msg.getTopic())); - this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, (String) null); + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, msg.getBrokerName()); } /** diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index bcfe29bd4f6..c1e3ee33dc1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.exception.MQBrokerException; @@ -43,6 +44,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.NetworkUtil; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -199,7 +201,7 @@ public long searchOffset(MessageQueue mq, long timestamp, BoundaryType boundaryT if (brokerAddr != null) { try { return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq, timestamp, - boundaryType, timeoutMillis); + boundaryType, timeoutMillis); } catch (Exception e) { throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); } @@ -277,13 +279,20 @@ public MessageExt viewMessage(String topic, String msgId) public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) throws MQClientException, InterruptedException { - return queryMessage(topic, key, maxNum, begin, end, false); + return queryMessage(null, topic, key, maxNum, begin, end, false); } public QueryResult queryMessageByUniqKey(String topic, String uniqKey, int maxNum, long begin, long end) throws MQClientException, InterruptedException { - return queryMessage(topic, uniqKey, maxNum, begin, end, true); + return queryMessage(null, topic, uniqKey, maxNum, begin, end, true); + } + + public QueryResult queryMessageByUniqKey(String clusterName, String topic, String uniqKey, int maxNum, long begin, + long end) + throws MQClientException, InterruptedException { + + return queryMessage(clusterName, topic, uniqKey, maxNum, begin, end, true); } public MessageExt queryMessageByUniqKey(String topic, @@ -311,25 +320,29 @@ public MessageExt queryMessageByUniqKey(String clusterName, String topic, } } - protected QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end, + public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end, boolean isUniqKey) throws MQClientException, InterruptedException { - return queryMessage(null, topic, key, maxNum, begin, end, isUniqKey); - } + boolean isLmq = MixAll.isLmq(topic); + + String routeTopic = topic; + // if topic is lmq ,then use clusterName as lmq parent topic + // Use clusterName or lmq parent topic to get topic route for lmq or rmq_sys_wheel_timer + if (!StringUtils.isEmpty(topic) && (isLmq || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) + && !StringUtils.isEmpty(clusterName)) { + routeTopic = clusterName; + } - protected QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end, - boolean isUniqKey) throws MQClientException, - InterruptedException { - TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic); + TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(routeTopic); if (null == topicRouteData) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); - topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(routeTopic); + topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(routeTopic); } if (topicRouteData != null) { List brokerAddrs = new LinkedList<>(); for (BrokerData brokerData : topicRouteData.getBrokerDatas()) { - if (clusterName != null && !clusterName.isEmpty() + if (!isLmq && clusterName != null && !clusterName.isEmpty() && !clusterName.equals(brokerData.getCluster())) { continue; } @@ -347,7 +360,11 @@ protected QueryResult queryMessage(String clusterName, String topic, String key, for (String addr : brokerAddrs) { try { QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader(); - requestHeader.setTopic(topic); + if (isLmq) { + requestHeader.setTopic(clusterName); + } else { + requestHeader.setTopic(topic); + } requestHeader.setKey(key); requestHeader.setMaxNum(maxNum); requestHeader.setBeginTimestamp(begin); @@ -436,7 +453,7 @@ public void operationFail(Throwable throwable) { String[] keyArray = keys.split(MessageConst.KEY_SEPARATOR); for (String k : keyArray) { // both topic and key must be equal at the same time - if (Objects.equals(key, k) && Objects.equals(topic, msgTopic)) { + if (Objects.equals(key, k) && (isLmq || Objects.equals(topic, msgTopic))) { matched = true; break; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 8a3d3dd0dcb..0a45f096235 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -78,6 +78,7 @@ import org.apache.rocketmq.common.namesrv.TopAddressing; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.common.utils.StartAndShutdown; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; @@ -112,6 +113,7 @@ import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData; import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo; import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; @@ -147,6 +149,7 @@ import org.apache.rocketmq.remoting.protocol.header.AddBrokerRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader; @@ -184,9 +187,9 @@ import org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader; import org.apache.rocketmq.remoting.protocol.header.GetTopicsByClusterRequestHeader; import org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader; import org.apache.rocketmq.remoting.protocol.header.LockBatchMqRequestHeader; import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader; @@ -247,7 +250,7 @@ import static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS; -public class MQClientAPIImpl implements NameServerUpdateCallback { +public class MQClientAPIImpl implements NameServerUpdateCallback, StartAndShutdown { private final static Logger log = LoggerFactory.getLogger(MQClientAPIImpl.class); private static boolean sendSmartMsg = Boolean.parseBoolean(System.getProperty("org.apache.rocketmq.client.sendSmartMsg", "true")); @@ -3016,6 +3019,19 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(final String brokerAddr, throw new MQClientException(response.getCode(), response.getRemark()); } + public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(final String brokerAddr, final String topic, final long timeoutMillis) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { + CheckRocksdbCqWriteProgressRequestHeader header = new CheckRocksdbCqWriteProgressRequestHeader(); + header.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, header); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (ResponseCode.SUCCESS == response.getCode()) { + return CheckRocksdbCqWriteProgressResponseBody.decode(response.getBody(), CheckRocksdbCqWriteProgressResponseBody.class); + } + throw new MQClientException(response.getCode(), response.getRemark()); + } + public void checkClientInBroker(final String brokerAddr, final String consumerGroup, final String clientId, final SubscriptionData subscriptionData, final long timeoutMillis) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index a3276cd7823..3f90b67ec99 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -164,10 +164,6 @@ private enum SubscriptionType { public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) { this.defaultLitePullConsumer = defaultLitePullConsumer; this.rpcHook = rpcHook; - this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( - this.defaultLitePullConsumer.getPullThreadNums(), - new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup()) - ); this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("MonitorMessageQueueChangeThread")); this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException(); } @@ -293,6 +289,8 @@ public synchronized void start() throws MQClientException { this.defaultLitePullConsumer.changeInstanceNameToPID(); } + initScheduledThreadPoolExecutor(); + initMQClientFactory(); initRebalanceImpl(); @@ -324,6 +322,13 @@ public synchronized void start() throws MQClientException { } } + private void initScheduledThreadPoolExecutor() { + this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( + this.defaultLitePullConsumer.getPullThreadNums(), + new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup()) + ); + } + private void initMQClientFactory() throws MQClientException { this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultLitePullConsumer, this.rpcHook); boolean registerOK = mQClientFactory.registerConsumer(this.defaultLitePullConsumer.getConsumerGroup(), this); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 0fef8666cb5..c92cadf5057 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -752,7 +752,7 @@ public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerN public void sendMessageBack(MessageExt msg, int delayLevel, final MessageQueue mq) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - sendMessageBack(msg, delayLevel, null, mq); + sendMessageBack(msg, delayLevel, msg.getBrokerName(), mq); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java index c68859b2889..0fa31b66406 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java @@ -26,6 +26,7 @@ import org.apache.rocketmq.client.common.NameserverAccessConfig; import org.apache.rocketmq.client.impl.ClientRemotingProcessor; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.utils.AsyncShutdownHelper; import org.apache.rocketmq.common.utils.StartAndShutdown; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.netty.NettyClientConfig; @@ -85,9 +86,11 @@ public void start() throws Exception { @Override public void shutdown() throws Exception { + AsyncShutdownHelper helper = new AsyncShutdownHelper(); for (int i = 0; i < this.clientNum; i++) { - clients[i].shutdown(); + helper.addTarget(clients[i]); } + helper.shutdown().await(Integer.MAX_VALUE, TimeUnit.SECONDS); } protected MQClientAPIExt createAndStart(String instanceName) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 0e70ee25951..74a2516174a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -194,6 +194,14 @@ public void setSemaphoreAsyncSendSize(int size) { semaphoreAsyncSendSize = new Semaphore(size, true); } + public int getSemaphoreAsyncSendNumAvailablePermits() { + return semaphoreAsyncSendNum == null ? 0 : semaphoreAsyncSendNum.availablePermits(); + } + + public int getSemaphoreAsyncSendSizeAvailablePermits() { + return semaphoreAsyncSendSize == null ? 0 : semaphoreAsyncSendSize.availablePermits(); + } + public void initTransactionEnv() { TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; if (producer.getExecutorService() != null) { @@ -563,7 +571,7 @@ public void run() { class BackpressureSendCallBack implements SendCallback { public boolean isSemaphoreAsyncSizeAcquired = false; - public boolean isSemaphoreAsyncNumbAcquired = false; + public boolean isSemaphoreAsyncNumAcquired = false; public int msgLen; private final SendCallback sendCallback; @@ -573,24 +581,49 @@ public BackpressureSendCallBack(final SendCallback sendCallback) { @Override public void onSuccess(SendResult sendResult) { - if (isSemaphoreAsyncSizeAcquired) { - semaphoreAsyncSendSize.release(msgLen); - } - if (isSemaphoreAsyncNumbAcquired) { - semaphoreAsyncSendNum.release(); - } + semaphoreProcessor(); sendCallback.onSuccess(sendResult); } @Override public void onException(Throwable e) { + semaphoreProcessor(); + sendCallback.onException(e); + } + + public void semaphoreProcessor() { if (isSemaphoreAsyncSizeAcquired) { + defaultMQProducer.acquireBackPressureForAsyncSendSizeLock(); semaphoreAsyncSendSize.release(msgLen); + defaultMQProducer.releaseBackPressureForAsyncSendSizeLock(); } - if (isSemaphoreAsyncNumbAcquired) { + if (isSemaphoreAsyncNumAcquired) { + defaultMQProducer.acquireBackPressureForAsyncSendNumLock(); semaphoreAsyncSendNum.release(); + defaultMQProducer.releaseBackPressureForAsyncSendNumLock(); } - sendCallback.onException(e); + } + + public void semaphoreAsyncAdjust(int semaphoreAsyncNum, int semaphoreAsyncSize) throws InterruptedException { + defaultMQProducer.acquireBackPressureForAsyncSendNumLock(); + if (semaphoreAsyncNum > 0) { + semaphoreAsyncSendNum.release(semaphoreAsyncNum); + } else { + semaphoreAsyncSendNum.acquire(- semaphoreAsyncNum); + } + defaultMQProducer.setBackPressureForAsyncSendNumInsideAdjust(defaultMQProducer.getBackPressureForAsyncSendNum() + + semaphoreAsyncNum); + defaultMQProducer.releaseBackPressureForAsyncSendNumLock(); + + defaultMQProducer.acquireBackPressureForAsyncSendSizeLock(); + if (semaphoreAsyncSize > 0) { + semaphoreAsyncSendSize.release(semaphoreAsyncSize); + } else { + semaphoreAsyncSendSize.acquire(- semaphoreAsyncSize); + } + defaultMQProducer.setBackPressureForAsyncSendSizeInsideAdjust(defaultMQProducer.getBackPressureForAsyncSendSize() + + semaphoreAsyncSize); + defaultMQProducer.releaseBackPressureForAsyncSendSizeLock(); } } @@ -599,32 +632,40 @@ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final throws MQClientException, InterruptedException { ExecutorService executor = this.getAsyncSenderExecutor(); boolean isEnableBackpressureForAsyncMode = this.getDefaultMQProducer().isEnableBackpressureForAsyncMode(); - boolean isSemaphoreAsyncNumbAcquired = false; + boolean isSemaphoreAsyncNumAcquired = false; boolean isSemaphoreAsyncSizeAcquired = false; int msgLen = msg.getBody() == null ? 1 : msg.getBody().length; + sendCallback.msgLen = msgLen; try { if (isEnableBackpressureForAsyncMode) { + defaultMQProducer.acquireBackPressureForAsyncSendNumLock(); long costTime = System.currentTimeMillis() - beginStartTime; - isSemaphoreAsyncNumbAcquired = timeout - costTime > 0 + + isSemaphoreAsyncNumAcquired = timeout - costTime > 0 && semaphoreAsyncSendNum.tryAcquire(timeout - costTime, TimeUnit.MILLISECONDS); - if (!isSemaphoreAsyncNumbAcquired) { + sendCallback.isSemaphoreAsyncNumAcquired = isSemaphoreAsyncNumAcquired; + defaultMQProducer.releaseBackPressureForAsyncSendNumLock(); + if (!isSemaphoreAsyncNumAcquired) { sendCallback.onException( new RemotingTooMuchRequestException("send message tryAcquire semaphoreAsyncNum timeout")); return; } + + defaultMQProducer.acquireBackPressureForAsyncSendSizeLock(); costTime = System.currentTimeMillis() - beginStartTime; + isSemaphoreAsyncSizeAcquired = timeout - costTime > 0 && semaphoreAsyncSendSize.tryAcquire(msgLen, timeout - costTime, TimeUnit.MILLISECONDS); + sendCallback.isSemaphoreAsyncSizeAcquired = isSemaphoreAsyncSizeAcquired; + defaultMQProducer.releaseBackPressureForAsyncSendSizeLock(); if (!isSemaphoreAsyncSizeAcquired) { sendCallback.onException( new RemotingTooMuchRequestException("send message tryAcquire semaphoreAsyncSize timeout")); return; } } - sendCallback.isSemaphoreAsyncSizeAcquired = isSemaphoreAsyncSizeAcquired; - sendCallback.isSemaphoreAsyncNumbAcquired = isSemaphoreAsyncNumbAcquired; - sendCallback.msgLen = msgLen; + executor.submit(runnable); } catch (RejectedExecutionException e) { if (isEnableBackpressureForAsyncMode) { diff --git a/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java b/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java new file mode 100644 index 00000000000..3d157313715 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.lock; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class ReadWriteCASLock { + //true : can lock ; false : not lock + private final AtomicBoolean writeLock = new AtomicBoolean(true); + + private final AtomicInteger readLock = new AtomicInteger(0); + + public void acquireWriteLock() { + boolean isLock = false; + do { + isLock = writeLock.compareAndSet(true, false); + } while (!isLock); + + do { + isLock = readLock.get() == 0; + } while (!isLock); + } + + public void releaseWriteLock() { + this.writeLock.compareAndSet(false, true); + } + + public void acquireReadLock() { + boolean isLock = false; + do { + isLock = writeLock.get(); + } while (!isLock); + readLock.getAndIncrement(); + } + + public void releaseReadLock() { + this.readLock.getAndDecrement(); + } + + public boolean getWriteLock() { + return this.writeLock.get() && this.readLock.get() == 0; + } + + public boolean getReadLock() { + return this.writeLock.get(); + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index 3ecd5987c35..f0842de8ba7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -24,6 +24,7 @@ import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.lock.ReadWriteCASLock; import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; import org.apache.rocketmq.client.trace.TraceDispatcher; import org.apache.rocketmq.client.trace.hook.EndTransactionTraceHookImpl; @@ -79,7 +80,8 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { ResponseCode.SYSTEM_BUSY, ResponseCode.NO_PERMISSION, ResponseCode.NO_BUYER_ID, - ResponseCode.NOT_IN_CURRENT_UNIT + ResponseCode.NOT_IN_CURRENT_UNIT, + ResponseCode.GO_AWAY )); /** @@ -174,6 +176,16 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { private RPCHook rpcHook = null; + /** + * backPressureForAsyncSendNum is guaranteed to be modified at runtime and no new requests are allowed + */ + private final ReadWriteCASLock backPressureForAsyncSendNumLock = new ReadWriteCASLock(); + + /** + * backPressureForAsyncSendSize is guaranteed to be modified at runtime and no new requests are allowed + */ + private final ReadWriteCASLock backPressureForAsyncSendSizeLock = new ReadWriteCASLock(); + /** * Compress level of compress algorithm. */ @@ -1333,18 +1345,64 @@ public int getBackPressureForAsyncSendNum() { return backPressureForAsyncSendNum; } + /** + * For user modify backPressureForAsyncSendNum at runtime + */ public void setBackPressureForAsyncSendNum(int backPressureForAsyncSendNum) { + this.backPressureForAsyncSendNumLock.acquireWriteLock(); + backPressureForAsyncSendNum = Math.max(backPressureForAsyncSendNum, 10); + int acquiredBackPressureForAsyncSendNum = this.backPressureForAsyncSendNum + - defaultMQProducerImpl.getSemaphoreAsyncSendNumAvailablePermits(); this.backPressureForAsyncSendNum = backPressureForAsyncSendNum; - defaultMQProducerImpl.setSemaphoreAsyncSendNum(backPressureForAsyncSendNum); + defaultMQProducerImpl.setSemaphoreAsyncSendNum(backPressureForAsyncSendNum - acquiredBackPressureForAsyncSendNum); + this.backPressureForAsyncSendNumLock.releaseWriteLock(); } public int getBackPressureForAsyncSendSize() { return backPressureForAsyncSendSize; } + /** + * For user modify backPressureForAsyncSendSize at runtime + */ public void setBackPressureForAsyncSendSize(int backPressureForAsyncSendSize) { + this.backPressureForAsyncSendSizeLock.acquireWriteLock(); + backPressureForAsyncSendSize = Math.max(backPressureForAsyncSendSize, 1024 * 1024); + int acquiredBackPressureForAsyncSendSize = this.backPressureForAsyncSendSize + - defaultMQProducerImpl.getSemaphoreAsyncSendSizeAvailablePermits(); + this.backPressureForAsyncSendSize = backPressureForAsyncSendSize; + defaultMQProducerImpl.setSemaphoreAsyncSendSize(backPressureForAsyncSendSize - acquiredBackPressureForAsyncSendSize); + this.backPressureForAsyncSendSizeLock.releaseWriteLock(); + } + + /** + * Used for system internal adjust backPressureForAsyncSendSize + */ + public void setBackPressureForAsyncSendSizeInsideAdjust(int backPressureForAsyncSendSize) { this.backPressureForAsyncSendSize = backPressureForAsyncSendSize; - defaultMQProducerImpl.setSemaphoreAsyncSendSize(backPressureForAsyncSendSize); + } + + /** + * Used for system internal adjust backPressureForAsyncSendNum + */ + public void setBackPressureForAsyncSendNumInsideAdjust(int backPressureForAsyncSendNum) { + this.backPressureForAsyncSendNum = backPressureForAsyncSendNum; + } + + public void acquireBackPressureForAsyncSendSizeLock() { + this.backPressureForAsyncSendSizeLock.acquireReadLock(); + } + + public void releaseBackPressureForAsyncSendSizeLock() { + this.backPressureForAsyncSendSizeLock.releaseReadLock(); + } + + public void acquireBackPressureForAsyncSendNumLock() { + this.backPressureForAsyncSendNumLock.acquireReadLock(); + } + + public void releaseBackPressureForAsyncSendNumLock() { + this.backPressureForAsyncSendNumLock.releaseReadLock(); } public List getTopics() { diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java index 6d62617eb8e..e321e1583d2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java @@ -302,14 +302,24 @@ public void run() { public void sendTraceData(List contextList) { Map> transBeanMap = new HashMap<>(16); - String currentRegionId; + String traceTopic; for (TraceContext context : contextList) { - currentRegionId = context.getRegionId(); + AccessChannel accessChannel = context.getAccessChannel(); + if (accessChannel == null) { + accessChannel = AsyncTraceDispatcher.this.accessChannel; + } + String currentRegionId = context.getRegionId(); if (currentRegionId == null || context.getTraceBeans().isEmpty()) { continue; } + if (AccessChannel.CLOUD == accessChannel) { + traceTopic = TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId; + } else { + traceTopic = traceTopicName; + } + String topic = context.getTraceBeans().get(0).getTopic(); - String key = topic + TraceConstants.CONTENT_SPLITOR + currentRegionId; + String key = topic + TraceConstants.CONTENT_SPLITOR + traceTopic; List transBeanList = transBeanMap.computeIfAbsent(key, k -> new ArrayList<>()); TraceTransferBean traceData = TraceDataEncoder.encoderFromContextBean(context); transBeanList.add(traceData); @@ -320,7 +330,7 @@ public void sendTraceData(List contextList) { } } - private void flushData(List transBeanList, String topic, String currentRegionId) { + private void flushData(List transBeanList, String topic, String traceTopic) { if (transBeanList.size() == 0) { return; } @@ -332,14 +342,14 @@ private void flushData(List transBeanList, String topic, Stri buffer.append(bean.getTransData()); count++; if (buffer.length() >= traceProducer.getMaxMessageSize()) { - sendTraceDataByMQ(keySet, buffer.toString(), TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId); + sendTraceDataByMQ(keySet, buffer.toString(), traceTopic); buffer.delete(0, buffer.length()); keySet.clear(); count = 0; } } if (count > 0) { - sendTraceDataByMQ(keySet, buffer.toString(), TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId); + sendTraceDataByMQ(keySet, buffer.toString(), traceTopic); } transBeanList.clear(); } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java index 65237bc8f76..592c247057b 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -63,8 +63,6 @@ import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.quality.Strictness; -import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; @@ -81,8 +79,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) -@MockitoSettings(strictness = Strictness.LENIENT) +@RunWith(MockitoJUnitRunner.Silent.class) public class DefaultLitePullConsumerTest { @Spy private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index a10fd74b34f..834be5cf16f 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -209,7 +209,9 @@ public PullResult answer(InvocationOnMock mock) throws Throwable { @AfterClass public static void terminate() { - pushConsumer.shutdown(); + if (pushConsumer != null) { + pushConsumer.shutdown(); + } } @Test diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java index 3663df24d65..f52aba2dc00 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java @@ -165,7 +165,7 @@ public void assertQueryMessage() throws InterruptedException, MQClientException, callback.operationSucceed(response); return null; }).when(mQClientAPIImpl).queryMessage(anyString(), any(), anyLong(), any(InvokeCallback.class), any()); - QueryResult actual = mqAdminImpl.queryMessage(defaultTopic, "keys", 100, 1L, 50L, false); + QueryResult actual = mqAdminImpl.queryMessage(defaultTopic, "keys", 100, 1L, 50L); assertNotNull(actual); assertEquals(1, actual.getMessageList().size()); assertEquals(defaultTopic, actual.getMessageList().get(0).getTopic()); diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java index 68563c02562..2bc9c5a18db 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java @@ -651,10 +651,11 @@ public void testQueryMessageByUniqKey() throws InterruptedException, MQClientExc @Test public void testSendMessageBack() throws InterruptedException, MQClientException, MQBrokerException, RemotingException { + when(mQClientFactory.findBrokerAddressInPublish(anyString())).thenReturn(defaultBrokerAddr); defaultMQPushConsumerImpl.sendMessageBack(createMessageExt(), 1, createMessageQueue()); verify(mqClientAPIImpl).consumerSendMessageBack( eq(defaultBrokerAddr), - any(), + eq(defaultBroker), any(MessageExt.class), any(), eq(1), diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 96086c7a255..4cf899f9708 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -551,6 +551,50 @@ public void testBatchSendMessageSync_Success() throws RemotingException, Interru producer.setAutoBatch(false); } + + @Test + public void testRunningSetBackCompress() throws RemotingException, InterruptedException, MQClientException { + final CountDownLatch countDownLatch = new CountDownLatch(5); + SendCallback sendCallback = new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + countDownLatch.countDown(); + } + + @Override + public void onException(Throwable e) { + e.printStackTrace(); + countDownLatch.countDown(); + } + }; + + // on enableBackpressureForAsyncMode + producer.setEnableBackpressureForAsyncMode(true); + producer.setBackPressureForAsyncSendNum(10); + producer.setBackPressureForAsyncSendSize(50 * 1024 * 1024); + Message message = new Message(); + message.setTopic("test"); + message.setBody("hello world".getBytes()); + MessageQueue mq = new MessageQueue("test", "BrokerA", 1); + //this message is send success + for (int i = 0; i < 5; i++) { + new Thread(new Runnable() { + @Override + public void run() { + try { + producer.send(message, mq, sendCallback); + } catch (MQClientException | RemotingException | InterruptedException e) { + throw new RuntimeException(e); + } + } + }).start(); + } + producer.setBackPressureForAsyncSendNum(15); + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(producer.defaultMQProducerImpl.getSemaphoreAsyncSendNumAvailablePermits() + countDownLatch.getCount()).isEqualTo(15); + producer.setEnableBackpressureForAsyncMode(false); + } + public static TopicRouteData createTopicRoute() { TopicRouteData topicRouteData = new TopicRouteData(); @@ -769,7 +813,7 @@ public void assertTotalBatchMaxBytes() throws NoSuchFieldException, IllegalAcces @Test public void assertGetRetryResponseCodes() { assertNotNull(producer.getRetryResponseCodes()); - assertEquals(7, producer.getRetryResponseCodes().size()); + assertEquals(8, producer.getRetryResponseCodes().size()); } @Test diff --git a/common/pom.xml b/common/pom.xml index 82994c9a197..b548d3df3c4 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 10bf7f76e86..2123e9b339d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -185,6 +185,11 @@ public class BrokerConfig extends BrokerIdentity { */ private int registerNameServerPeriod = 1000 * 30; + /** + * This configurable item defines interval of update name server address. Default: 120 * 1000 milliseconds + */ + private int updateNameServerAddrPeriod = 1000 * 120; + /** * the interval to send heartbeat to name server for liveness detection. */ @@ -226,7 +231,7 @@ public class BrokerConfig extends BrokerIdentity { // read message from pop retry topic v1, for the compatibility, will be removed in the future version private boolean retrieveMessageFromPopRetryTopicV1 = true; private boolean enableRetryTopicV2 = false; - + private int popFromRetryProbability = 20; private boolean realTimeNotifyConsumerChange = true; private boolean litePullMessageEnable = true; @@ -558,6 +563,15 @@ public void setEnablePopLog(boolean enablePopLog) { this.enablePopLog = enablePopLog; } + public int getPopFromRetryProbability() { + return popFromRetryProbability; + } + + public void setPopFromRetryProbability(int popFromRetryProbability) { + this.popFromRetryProbability = popFromRetryProbability; + } + + public boolean isTraceOn() { return traceOn; } @@ -1837,4 +1851,12 @@ public boolean isSkipWhenCKRePutReachMaxTimes() { public void setSkipWhenCKRePutReachMaxTimes(boolean skipWhenCKRePutReachMaxTimes) { this.skipWhenCKRePutReachMaxTimes = skipWhenCKRePutReachMaxTimes; } + + public int getUpdateNameServerAddrPeriod() { + return updateNameServerAddrPeriod; + } + + public void setUpdateNameServerAddrPeriod(int updateNameServerAddrPeriod) { + this.updateNameServerAddrPeriod = updateNameServerAddrPeriod; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 8ac75a72c98..a03668e51ce 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ public class MQVersion { - public static final int CURRENT_VERSION = Version.V5_3_0.ordinal(); + public static final int CURRENT_VERSION = Version.V5_3_1.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index f88b8e198bf..13522889bb3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -17,6 +17,15 @@ package org.apache.rocketmq.common.config; import com.google.common.collect.Maps; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.DataConverter; @@ -40,16 +49,6 @@ import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - import static org.rocksdb.RocksDB.NOT_FOUND; public abstract class AbstractRocksDBStorage { @@ -495,7 +494,9 @@ public void statRocksdb(Logger logger) { String blocksPinnedByIteratorMemUsage = this.db.getProperty("rocksdb.block-cache-pinned-usage"); logger.info("MemUsage. blockCache: {}, indexesAndFilterBlock: {}, memtable: {}, blocksPinnedByIterator: {}", blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage); - } catch (Exception ignored) { + } catch (Exception e) { + logger.error("statRocksdb Failed. {}", this.dbPath, e); + throw new RuntimeException(e); } } } diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java b/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java new file mode 100644 index 00000000000..da765d5e749 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class AsyncShutdownHelper { + private final AtomicBoolean shutdown; + private final List targetList; + + private CountDownLatch countDownLatch; + + public AsyncShutdownHelper() { + this.targetList = new ArrayList<>(); + this.shutdown = new AtomicBoolean(false); + } + + public void addTarget(Shutdown target) { + if (shutdown.get()) { + return; + } + targetList.add(target); + } + + public AsyncShutdownHelper shutdown() { + if (shutdown.get()) { + return this; + } + if (targetList.isEmpty()) { + return this; + } + this.countDownLatch = new CountDownLatch(targetList.size()); + for (Shutdown target : targetList) { + Runnable runnable = () -> { + try { + target.shutdown(); + } catch (Exception ignored) { + + } finally { + countDownLatch.countDown(); + } + }; + new Thread(runnable).start(); + } + return this; + } + + public boolean await(long time, TimeUnit unit) throws InterruptedException { + if (shutdown.get()) { + return false; + } + try { + return this.countDownLatch.await(time, unit); + } finally { + shutdown.compareAndSet(false, true); + } + } +} diff --git a/container/pom.xml b/container/pom.xml index b9514defdb8..cc177abeea9 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -18,7 +18,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/controller/pom.xml b/controller/pom.xml index 82b6fc7d969..7092ca2b3cd 100644 --- a/controller/pom.xml +++ b/controller/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 jar diff --git a/distribution/pom.xml b/distribution/pom.xml index 60fc6170bbe..88521fbede7 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/docs/cn/best_practice.md b/docs/cn/best_practice.md index 5cc5b37643f..36d6acff6bd 100755 --- a/docs/cn/best_practice.md +++ b/docs/cn/best_practice.md @@ -253,7 +253,7 @@ DefaultMQProducer、TransactionMQProducer、DefaultMQPushConsumer、DefaultMQPul | clientIP | 本机IP | 客户端本机IP地址，某些机器会发生无法识别客户端IP地址情况，需要应用在代码中强制指定 | | instanceName | DEFAULT | 客户端实例名称，客户端创建的多个Producer、Consumer实际是共用一个内部实例（这个实例包含网络连接、线程资源等） | | clientCallbackExecutorThreads | 4 | 通信层异步回调线程数 | -| pollNameServerInteval | 30000 | 轮询Name Server间隔时间，单位毫秒 | +| pollNameServerInterval | 30000 | 轮询Name Server间隔时间，单位毫秒 | | heartbeatBrokerInterval | 30000 | 向Broker发送心跳间隔时间，单位毫秒 | | persistConsumerOffsetInterval | 5000 | 持久化Consumer消费进度间隔时间，单位毫秒 | diff --git a/docs/en/Configuration_Client.md b/docs/en/Configuration_Client.md index 4d999b2feda..4679957af5a 100644 --- a/docs/en/Configuration_Client.md +++ b/docs/en/Configuration_Client.md @@ -48,7 +48,7 @@ HTTP static server addressing is recommended, because it is simple client deploy | clientIP | local IP | Client local ip address, some machines will fail to recognize the client IP address, which needs to be enforced in the code | | instanceName | DEFAULT | Name of the client instance, Multiple producers and consumers created by the client actually share one internal instance (this instance contains network connection, thread resources, etc.). | | clientCallbackExecutorThreads | 4 | Number of communication layer asynchronous callback threads | -| pollNameServerInteval | 30000 | Polling the Name Server interval in milliseconds | +| pollNameServerInterval | 30000 | Polling the Name Server interval in milliseconds | | heartbeatBrokerInterval | 30000 | The heartbeat interval, in milliseconds, is sent to the Broker | | persistConsumerOffsetInterval | 5000 | The persistent Consumer consumes the progress interval in milliseconds | diff --git a/example/pom.xml b/example/pom.xml index 7685a811690..19047c2f552 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java similarity index 97% rename from example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java rename to example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java index 81ef2e13859..5fee9480287 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.simple; +package org.apache.rocketmq.example.lmq; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; @@ -47,6 +47,7 @@ public static void main(String[] args) throws MQClientException, InterruptedExce for (int i = 0; i < 128; i++) { try { Message msg = new Message(TOPIC, TAG, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); + msg.setKeys("Key" + i); msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH /* "INNER_MULTI_DISPATCH" */, String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */); SendResult sendResult = producer.send(msg); diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java similarity index 98% rename from example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java rename to example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java index 7b1bdc39215..931dd96b48f 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.simple; +package org.apache.rocketmq.example.lmq; import java.util.Arrays; import java.util.HashSet; diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java similarity index 98% rename from example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java rename to example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java index efe37d86816..f8926a05dfd 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.simple; +package org.apache.rocketmq.example.lmq; import com.google.common.collect.Lists; diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java similarity index 99% rename from example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java rename to example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java index 2044057b2af..517eb12b7d2 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.simple; +package org.apache.rocketmq.example.lmq; import com.google.common.collect.Lists; import java.util.HashMap; diff --git a/filter/pom.xml b/filter/pom.xml index 0acaa73f8ae..262177b61c2 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index d53540601e6..012ebafe064 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 09ab5ed2586..8ea4745b25d 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 41fc3db0c40..ab4f9c45f67 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 2012 org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -526,6 +526,21 @@ https://builds.apache.org/analysis + + skip-unit-tests + + + + maven-surefire-plugin + ${maven-surefire-plugin.version} + + true + true + + + + + diff --git a/proxy/pom.xml b/proxy/pom.xml index 41e6fa95f55..e608d9f587f 100644 --- a/proxy/pom.xml +++ b/proxy/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java index 39d7057bddd..518868831f4 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java @@ -118,11 +118,11 @@ protected HAProxyMessage buildHAProxyMessage(Channel inboundChannel) throws Ille } } else { String remoteAddr = RemotingHelper.parseChannelRemoteAddr(inboundChannel); - sourceAddress = StringUtils.substringBefore(remoteAddr, CommonConstants.COLON); + sourceAddress = StringUtils.substringBeforeLast(remoteAddr, CommonConstants.COLON); sourcePort = Integer.parseInt(StringUtils.substringAfterLast(remoteAddr, CommonConstants.COLON)); String localAddr = RemotingHelper.parseChannelLocalAddr(inboundChannel); - destinationAddress = StringUtils.substringBefore(localAddr, CommonConstants.COLON); + destinationAddress = StringUtils.substringBeforeLast(localAddr, CommonConstants.COLON); destinationPort = Integer.parseInt(StringUtils.substringAfterLast(localAddr, CommonConstants.COLON)); } diff --git a/remoting/pom.xml b/remoting/pom.xml index 566c983ea98..65e9a852fcc 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 6f61e75e01a..ffa37260594 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -39,8 +39,8 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.rocketmq.common.AbortProcessException; @@ -278,6 +278,7 @@ public void processRequestCommand(final ChannelHandlerContext ctx, final Remotin "please go away"); response.setOpaque(opaque); writeResponse(ctx.channel(), cmd, response); + log.info("proxy is shutting down, write response GO_AWAY. channel={}, requestCode={}, opaque={}", ctx.channel(), cmd.getCode(), opaque); return; } } @@ -392,7 +393,7 @@ public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cm responseFuture.release(); } } else { - log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + log.warn("receive response, cmd={}, but not matched any request, address={}, channelId={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), ctx.channel().id()); } } @@ -559,13 +560,13 @@ public void operationFail(Throwable throwable) { return; } requestFail(opaque); - log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); + log.warn("send a request command to channel <{}>, channelId={}, failed.", RemotingHelper.parseChannelRemoteAddr(channel), channel.id()); }); return future; } catch (Exception e) { responseTable.remove(opaque); responseFuture.release(); - log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); + log.warn("send a request command to channel <{}> channelId={} Exception", RemotingHelper.parseChannelRemoteAddr(channel), channel.id(), e); future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e)); return future; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 41976122b2f..ae82b09edaf 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -49,7 +49,6 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.security.cert.CertificateException; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -416,14 +415,14 @@ public void closeChannel(final String addr, final Channel channel) { boolean removeItemFromTable = true; final ChannelWrapper prevCW = this.channelTables.get(addrRemote); - LOGGER.info("closeChannel: begin close the channel[{}] Found: {}", addrRemote, prevCW != null); + LOGGER.info("closeChannel: begin close the channel[addr={}, id={}] Found: {}", addrRemote, channel.id(), prevCW != null); if (null == prevCW) { - LOGGER.info("closeChannel: the channel[{}] has been removed from the channel table before", addrRemote); + LOGGER.info("closeChannel: the channel[addr={}, id={}] has been removed from the channel table before", addrRemote, channel.id()); removeItemFromTable = false; } else if (prevCW.isWrapperOf(channel)) { - LOGGER.info("closeChannel: the channel[{}] has been closed before, and has been created again, nothing to do.", - addrRemote); + LOGGER.info("closeChannel: the channel[addr={}, id={}] has been closed before, and has been created again, nothing to do.", + addrRemote, channel.id()); removeItemFromTable = false; } @@ -432,7 +431,7 @@ public void closeChannel(final String addr, final Channel channel) { if (channelWrapper != null && channelWrapper.tryClose(channel)) { this.channelTables.remove(addrRemote); } - LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + LOGGER.info("closeChannel: the channel[addr={}, id={}] was removed from channel table", addrRemote, channel.id()); } RemotingHelper.closeChannel(channel); @@ -471,7 +470,7 @@ public void closeChannel(final Channel channel) { } if (null == prevCW) { - LOGGER.info("eventCloseChannel: the channel[{}] has been removed from the channel table before", addrRemote); + LOGGER.info("eventCloseChannel: the channel[addr={}, id={}] has been removed from the channel table before", RemotingHelper.parseChannelRemoteAddr(channel), channel.id()); removeItemFromTable = false; } @@ -480,11 +479,11 @@ public void closeChannel(final Channel channel) { if (channelWrapper != null && channelWrapper.tryClose(channel)) { this.channelTables.remove(addrRemote); } - LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + LOGGER.info("closeChannel: the channel[addr={}, id={}] was removed from channel table", addrRemote, channel.id()); RemotingHelper.closeChannel(channel); } } catch (Exception e) { - LOGGER.error("closeChannel: close the channel exception", e); + LOGGER.error("closeChannel: close the channel[id={}] exception", channel.id(), e); } finally { this.lockChannelTables.unlock(); } @@ -521,10 +520,11 @@ public void updateNameServerAddressList(List addrs) { this.namesrvAddrList.set(addrs); // should close the channel if choosed addr is not exist. - if (this.namesrvAddrChoosed.get() != null && !addrs.contains(this.namesrvAddrChoosed.get())) { - String namesrvAddr = this.namesrvAddrChoosed.get(); + String chosenNameServerAddr = this.namesrvAddrChoosed.get(); + if (chosenNameServerAddr != null && !addrs.contains(chosenNameServerAddr)) { + namesrvAddrChoosed.compareAndSet(chosenNameServerAddr, null); for (String addr : this.channelTables.keySet()) { - if (addr.contains(namesrvAddr)) { + if (addr.contains(chosenNameServerAddr)) { ChannelWrapper channelWrapper = this.channelTables.get(addr); if (channelWrapper != null) { channelWrapper.close(); @@ -562,9 +562,9 @@ public RemotingCommand invokeSync(String addr, final RemotingCommand request, lo boolean shouldClose = left > MIN_CLOSE_TIMEOUT_MILLIS || left > timeoutMillis / 4; if (nettyClientConfig.isClientCloseSocketIfTimeout() && shouldClose) { this.closeChannel(addr, channel); - LOGGER.warn("invokeSync: close socket because of timeout, {}ms, {}", timeoutMillis, channelRemoteAddr); + LOGGER.warn("invokeSync: close socket because of timeout, {}ms, channel[addr={}, id={}]", timeoutMillis, channelRemoteAddr, channel.id()); } - LOGGER.warn("invokeSync: wait response timeout exception, the channel[{}]", channelRemoteAddr); + LOGGER.warn("invokeSync: wait response timeout exception, the channel[addr={}, id={}]", channelRemoteAddr, channel.id()); throw e; } } else { @@ -819,10 +819,11 @@ public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand response = responseFuture.getResponseCommand(); if (response.getCode() == ResponseCode.GO_AWAY) { if (nettyClientConfig.isEnableReconnectForGoAway()) { + LOGGER.info("Receive go away from channelId={}, channel={}", channel.id(), channel); ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> { try { - if (channelWrapper0.reconnect()) { - LOGGER.info("Receive go away from channel {}, recreate the channel", channel0); + if (channelWrapper0.reconnect(channel0)) { + LOGGER.info("Receive go away from channelId={}, channel={}, recreate the channelId={}", channel0.id(), channel0, channelWrapper0.getChannel().id()); channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0); } } catch (Throwable t) { @@ -830,10 +831,11 @@ public CompletableFuture invokeImpl(final Channel channel, final } return channelWrapper0; }); - if (channelWrapper != null) { + if (channelWrapper != null && !channelWrapper.isWrapperOf(channel)) { if (nettyClientConfig.isEnableTransparentRetry()) { RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); retryRequest.setBody(request.getBody()); + retryRequest.setExtFields(request.getExtFields()); if (channelWrapper.isOK()) { long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); stopwatch.stop(); @@ -865,6 +867,8 @@ public CompletableFuture invokeImpl(final Channel channel, final return future; } } + } else { + LOGGER.warn("invokeImpl receive GO_AWAY, channelWrapper is null or channel is the same in wrapper, channelId={}", channel.id()); } } } @@ -1002,7 +1006,6 @@ class ChannelWrapper { // only affected by sync or async request, oneway is not included. private ChannelFuture channelToClose; private long lastResponseTime; - private volatile long lastReconnectTimestamp = 0L; private final String channelAddress; public ChannelWrapper(String address, ChannelFuture channelFuture) { @@ -1021,10 +1024,7 @@ public boolean isWritable() { } public boolean isWrapperOf(Channel channel) { - if (this.channelFuture.channel() != null && this.channelFuture.channel() == channel) { - return true; - } - return false; + return this.channelFuture.channel() != null && this.channelFuture.channel() == channel; } private Channel getChannel() { @@ -1052,20 +1052,27 @@ public String getChannelAddress() { return channelAddress; } - public boolean reconnect() { + public boolean reconnect(Channel channel) { + if (!isWrapperOf(channel)) { + LOGGER.warn("channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}",getChannel().id(), channel.id()); + return false; + } if (lock.writeLock().tryLock()) { try { - if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) { + if (isWrapperOf(channel)) { channelToClose = channelFuture; String[] hostAndPort = getHostAndPort(channelAddress); channelFuture = fetchBootstrap(channelAddress) .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); - lastReconnectTimestamp = System.currentTimeMillis(); return true; + } else { + LOGGER.warn("channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}",getChannel().id(), channel.id()); } } finally { lock.writeLock().unlock(); } + } else { + LOGGER.warn("channelWrapper reconnect try lock fail, now channelId={}", getChannel().id()); } return false; } @@ -1152,7 +1159,7 @@ public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, Sock @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.info("NETTY CLIENT PIPELINE: ACTIVE, {}", remoteAddress); + LOGGER.info("NETTY CLIENT PIPELINE: ACTIVE, {}, channelId={}", remoteAddress, ctx.channel().id()); super.channelActive(ctx); if (NettyRemotingClient.this.channelEventListener != null) { @@ -1175,7 +1182,7 @@ public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws @Override public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.info("NETTY CLIENT PIPELINE: CLOSE {}", remoteAddress); + LOGGER.info("NETTY CLIENT PIPELINE: CLOSE channel[addr={}, id={}]", remoteAddress, ctx.channel().id()); closeChannel(ctx.channel()); super.close(ctx, promise); NettyRemotingClient.this.failFast(ctx.channel()); @@ -1187,7 +1194,7 @@ public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exce @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.info("NETTY CLIENT PIPELINE: channelInactive, the channel[{}]", remoteAddress); + LOGGER.info("NETTY CLIENT PIPELINE: channelInactive, the channel[addr={}, id={}]", remoteAddress, ctx.channel().id()); closeChannel(ctx.channel()); super.channelInactive(ctx); } @@ -1198,7 +1205,7 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc IdleStateEvent event = (IdleStateEvent) evt; if (event.state().equals(IdleState.ALL_IDLE)) { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception [{}]", remoteAddress); + LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception channel[addr={}, id={}]", remoteAddress, ctx.channel().id()); closeChannel(ctx.channel()); if (NettyRemotingClient.this.channelEventListener != null) { NettyRemotingClient.this @@ -1213,8 +1220,7 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught {}", remoteAddress); - LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught exception.", cause); + LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught channel[addr={}, id={}]", remoteAddress, ctx.channel().id(), cause); closeChannel(ctx.channel()); if (NettyRemotingClient.this.channelEventListener != null) { NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress, ctx.channel())); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index 51f8b85009e..cbf25c23c60 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -270,8 +270,9 @@ public void run(Timeout timeout) { */ protected ChannelPipeline configChannel(SocketChannel ch) { return ch.pipeline() - .addLast(defaultEventExecutorGroup, HANDSHAKE_HANDLER_NAME, new HandshakeHandler()) - .addLast(defaultEventExecutorGroup, + .addLast(nettyServerConfig.isServerNettyWorkerGroupEnable() ? defaultEventExecutorGroup : null, + HANDSHAKE_HANDLER_NAME, new HandshakeHandler()) + .addLast(nettyServerConfig.isServerNettyWorkerGroupEnable() ? defaultEventExecutorGroup : null, encoder, new NettyDecoder(), distributionHandler, @@ -782,16 +783,16 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception private void handleWithMessage(HAProxyMessage msg, Channel channel) { try { if (StringUtils.isNotBlank(msg.sourceAddress())) { - channel.attr(AttributeKeys.PROXY_PROTOCOL_ADDR).set(msg.sourceAddress()); + RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_ADDR, msg.sourceAddress()); } if (msg.sourcePort() > 0) { - channel.attr(AttributeKeys.PROXY_PROTOCOL_PORT).set(String.valueOf(msg.sourcePort())); + RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_PORT, String.valueOf(msg.sourcePort())); } if (StringUtils.isNotBlank(msg.destinationAddress())) { - channel.attr(AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR).set(msg.destinationAddress()); + RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR, msg.destinationAddress()); } if (msg.destinationPort() > 0) { - channel.attr(AttributeKeys.PROXY_PROTOCOL_SERVER_PORT).set(String.valueOf(msg.destinationPort())); + RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_SERVER_PORT, String.valueOf(msg.destinationPort())); } if (CollectionUtils.isNotEmpty(msg.tlvs())) { msg.tlvs().forEach(tlv -> { @@ -811,6 +812,6 @@ protected void handleHAProxyTLV(HAProxyTLV tlv, Channel channel) { } AttributeKey key = AttributeKeys.valueOf( HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); - channel.attr(key).set(new String(valueBytes, CharsetUtil.UTF_8)); + RemotingHelper.setPropertyToAttr(channel, key, new String(valueBytes, CharsetUtil.UTF_8)); } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java index 6564404b920..664dee8371c 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java @@ -36,6 +36,7 @@ public class NettyServerConfig implements Cloneable { private int writeBufferHighWaterMark = NettySystemConfig.writeBufferHighWaterMark; private int writeBufferLowWaterMark = NettySystemConfig.writeBufferLowWaterMark; private int serverSocketBacklog = NettySystemConfig.socketBacklog; + private boolean serverNettyWorkerGroupEnable = true; private boolean serverPooledByteBufAllocatorEnable = true; private boolean enableShutdownGracefully = false; @@ -175,6 +176,14 @@ public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) { this.writeBufferHighWaterMark = writeBufferHighWaterMark; } + public boolean isServerNettyWorkerGroupEnable() { + return serverNettyWorkerGroupEnable; + } + + public void setServerNettyWorkerGroupEnable(boolean serverNettyWorkerGroupEnable) { + this.serverNettyWorkerGroupEnable = serverNettyWorkerGroupEnable; + } + public boolean isEnableShutdownGracefully() { return enableShutdownGracefully; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java index f45ff6fa484..cfc5cc22785 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java @@ -217,6 +217,7 @@ public class RequestCode { public static final int GET_SUBSCRIPTIONGROUP_CONFIG = 352; public static final int UPDATE_AND_GET_GROUP_FORBIDDEN = 353; + public static final int CHECK_ROCKSDB_CQ_WRITE_PROGRESS = 354; public static final int LITE_PULL_MESSAGE = 361; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java new file mode 100644 index 00000000000..76719ac1a24 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.remoting.protocol.body; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class CheckRocksdbCqWriteProgressResponseBody extends RemotingSerializable { + + String diffResult; + + public String getDiffResult() { + return diffResult; + } + + public void setDiffResult(String diffResult) { + this.diffResult = diffResult; + } + + +} diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java new file mode 100644 index 00000000000..fee158b4976 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting.protocol.header; + +import org.apache.rocketmq.common.action.Action; +import org.apache.rocketmq.common.action.RocketMQAction; +import org.apache.rocketmq.common.resource.ResourceType; +import org.apache.rocketmq.common.resource.RocketMQResource; +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RequestCode; + +@RocketMQAction(value = RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, action = Action.GET) +public class CheckRocksdbCqWriteProgressRequestHeader implements CommandCustomHeader { + + @CFNotNull + @RocketMQResource(ResourceType.TOPIC) + private String topic; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 562a5ea2a33..f6c5b3f54d6 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 6de01626772..d49de5ae267 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index f707d8fbd87..972e71aadd8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -61,6 +61,7 @@ import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; import org.apache.rocketmq.store.logfile.MappedFile; +import org.apache.rocketmq.store.queue.MultiDispatchUtils; import org.apache.rocketmq.store.util.LibC; import org.rocksdb.RocksDBException; @@ -1834,12 +1835,13 @@ class DefaultAppendMessageCallback implements AppendMessageCallback { private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; // Store the message content private final ByteBuffer msgStoreItemMemory; - private final int crc32ReservedLength = enabledAppendPropCRC ? CommitLog.CRC32_RESERVED_LEN : 0; + private final int crc32ReservedLength; private final MessageStoreConfig messageStoreConfig; DefaultAppendMessageCallback(MessageStoreConfig messageStoreConfig) { this.msgStoreItemMemory = ByteBuffer.allocate(END_FILE_MIN_BLANK_LENGTH); this.messageStoreConfig = messageStoreConfig; + this.crc32ReservedLength = messageStoreConfig.isEnabledAppendPropCRC() ? CommitLog.CRC32_RESERVED_LEN : 0; } public AppendMessageResult handlePropertiesForLmqMsg(ByteBuffer preEncodeBuffer, @@ -1902,7 +1904,7 @@ public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET
ByteBuffer preEncodeBuffer = msgInner.getEncodedBuff(); - boolean isMultiDispatchMsg = messageStoreConfig.isEnableMultiDispatch() && CommitLog.isMultiDispatchMsg(msgInner); + final boolean isMultiDispatchMsg = CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner); if (isMultiDispatchMsg) { AppendMessageResult appendMessageResult = handlePropertiesForLmqMsg(preEncodeBuffer, msgInner); if (appendMessageResult != null) { @@ -2243,8 +2245,9 @@ public FlushManager getFlushManager() { return flushManager; } - public static boolean isMultiDispatchMsg(MessageExtBrokerInner msg) { - return StringUtils.isNoneBlank(msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) && !msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX); + public static boolean isMultiDispatchMsg(MessageStoreConfig messageStoreConfig, MessageExtBrokerInner msg) { + return StringUtils.isNotBlank(msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) && + MultiDispatchUtils.isNeedHandleMultiDispatch(messageStoreConfig, msg.getTopic()); } private boolean isCloseReadAhead() { diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index f159c31a7be..8b46c7f5ce4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -163,11 +163,13 @@ public class DefaultMessageStore implements MessageStore { private volatile boolean shutdown = true; protected boolean notifyMessageArriveInBatch = false; - private StoreCheckpoint storeCheckpoint; + protected StoreCheckpoint storeCheckpoint; private TimerMessageStore timerMessageStore; private final LinkedList dispatcherList; + private RocksDBMessageStore rocksDBMessageStore; + private RandomAccessFile lockFile; private FileLock lock; @@ -354,12 +356,7 @@ public boolean load() { } if (result) { - this.storeCheckpoint = - new StoreCheckpoint( - StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir())); - this.masterFlushedOffset = this.storeCheckpoint.getMasterFlushedOffset(); - setConfirmOffset(this.storeCheckpoint.getConfirmPhyOffset()); - + loadCheckPoint(); result = this.indexService.load(lastExitOK); this.recover(lastExitOK); LOGGER.info("message store recover end, and the max phy offset = {}", this.getMaxPhyOffset()); @@ -381,6 +378,14 @@ public boolean load() { return result; } + public void loadCheckPoint() throws IOException { + this.storeCheckpoint = + new StoreCheckpoint( + StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir())); + this.masterFlushedOffset = this.storeCheckpoint.getMasterFlushedOffset(); + setConfirmOffset(this.storeCheckpoint.getConfirmPhyOffset()); + } + /** * @throws Exception */ @@ -511,6 +516,10 @@ public void shutdown() { this.compactionService.shutdown(); } + if (messageStoreConfig.isRocksdbCQDoubleWriteEnable()) { + this.rocksDBMessageStore.consumeQueueStore.shutdown(); + } + this.flushConsumeQueueService.shutdown(); this.allocateMappedFileService.shutdown(); this.storeCheckpoint.flush(); @@ -985,7 +994,7 @@ public long getMaxOffsetInQueue(String topic, int queueId) { @Override public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { if (committed) { - ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); + ConsumeQueueInterface logic = this.getConsumeQueue(topic, queueId); if (logic != null) { return logic.getMaxOffsetInQueue(); } @@ -1021,7 +1030,7 @@ public void setTimerMessageStore(TimerMessageStore timerMessageStore) { @Override public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) { - ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); + ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId); if (consumeQueue != null) { CqUnit cqUnit = consumeQueue.get(consumeQueueOffset); if (cqUnit != null) { @@ -1157,7 +1166,7 @@ public boolean getLastMappedFile(long startOffset) { @Override public long getEarliestMessageTime(String topic, int queueId) { - ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId); if (logicQueue != null) { Pair pair = logicQueue.getEarliestUnitAndStoreTime(); if (pair != null && pair.getObject2() != null) { @@ -1189,7 +1198,7 @@ public long getEarliestMessageTime() { @Override public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) { - ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId); if (logicQueue != null) { Pair pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset); if (pair != null && pair.getObject2() != null) { @@ -1207,12 +1216,12 @@ public CompletableFuture getMessageStoreTimeStampAsync(String topic, int q @Override public long getMessageTotalInQueue(String topic, int queueId) { - ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId); if (logicQueue != null) { return logicQueue.getMessageTotalInQueue(); } - return -1; + return 0; } @Override @@ -1496,7 +1505,7 @@ public boolean checkInDiskByConsumeOffset(final String topic, final int queueId, final long maxOffsetPy = this.commitLog.getMaxOffset(); - ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); + ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId); if (consumeQueue != null) { CqUnit cqUnit = consumeQueue.get(consumeOffset); @@ -1512,7 +1521,7 @@ public boolean checkInDiskByConsumeOffset(final String topic, final int queueId, @Override public boolean checkInMemByConsumeOffset(final String topic, final int queueId, long consumeOffset, int batchSize) { - ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); + ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId); if (consumeQueue != null) { CqUnit firstCQItem = consumeQueue.get(consumeOffset); if (firstCQItem == null) { @@ -3251,6 +3260,17 @@ public HARuntimeInfo getHARuntimeInfo() { } } + public void enableRocksdbCQWrite() { + try { + RocksDBMessageStore store = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, this.topicConfigTable); + this.rocksDBMessageStore = store; + store.loadAndStartConsumerServiceOnly(); + addDispatcher(store.getDispatcherBuildRocksdbConsumeQueue()); + } catch (Exception e) { + LOGGER.error("enableRocksdbCqWrite error", e); + } + } + public int getMaxDelayLevel() { return maxDelayLevel; } @@ -3338,4 +3358,12 @@ public boolean isTransientStorePoolEnable() { public long getReputFromOffset() { return this.reputMessageService.getReputFromOffset(); } + + public RocksDBMessageStore getRocksDBMessageStore() { + return this.rocksDBMessageStore; + } + + public ConsumeQueueStoreInterface getConsumeQueueStore() { + return consumeQueueStore; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java index 20e9a652b7e..5c74918d9e6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java @@ -175,7 +175,7 @@ public PutMessageResult encodeWithoutProperties(MessageExtBrokerInner msgInner) public PutMessageResult encode(MessageExtBrokerInner msgInner) { this.byteBuf.clear(); - if (messageStoreConfig.isEnableMultiDispatch() && CommitLog.isMultiDispatchMsg(msgInner)) { + if (CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner)) { return encodeWithoutProperties(msgInner); } diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java index 6141b778bf7..90df7aed596 100644 --- a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java @@ -16,16 +16,16 @@ */ package org.apache.rocketmq.store; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.Meter; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Supplier; - -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.Meter; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; @@ -39,6 +39,8 @@ public class RocksDBMessageStore extends DefaultMessageStore { + private CommitLogDispatcherBuildRocksdbConsumeQueue dispatcherBuildRocksdbConsumeQueue; + public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws IOException { @@ -178,4 +180,40 @@ public void initMetrics(Meter meter, Supplier attributesBuild // Also add some metrics for rocksdb's monitoring. RocksDBStoreMetricsManager.init(meter, attributesBuilderSupplier, this); } + + public CommitLogDispatcherBuildRocksdbConsumeQueue getDispatcherBuildRocksdbConsumeQueue() { + return dispatcherBuildRocksdbConsumeQueue; + } + + class CommitLogDispatcherBuildRocksdbConsumeQueue implements CommitLogDispatcher { + @Override + public void dispatch(DispatchRequest request) throws RocksDBException { + final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag()); + switch (tranType) { + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + putMessagePositionInfo(request); + break; + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + break; + } + } + } + + public void loadAndStartConsumerServiceOnly() { + try { + this.dispatcherBuildRocksdbConsumeQueue = new CommitLogDispatcherBuildRocksdbConsumeQueue(); + boolean loadResult = this.consumeQueueStore.load(); + if (!loadResult) { + throw new RuntimeException("load consume queue failed"); + } + super.loadCheckPoint(); + this.consumeQueueStore.start(); + } catch (Exception e) { + ERROR_LOG.error("loadAndStartConsumerServiceOnly error", e); + throw new RuntimeException(e); + } + } + } diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 0b45d92418e..68531284389 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -424,6 +424,37 @@ public class MessageStoreConfig { private boolean putConsumeQueueDataByFileChannel = true; + private boolean transferOffsetJsonToRocksdb = false; + + private boolean rocksdbCQDoubleWriteEnable = false; + + private int batchWriteKvCqSize = 16; + + + public int getBatchWriteKvCqSize() { + return batchWriteKvCqSize; + } + + public void setBatchWriteKvCqSize(int batchWriteKvCqSize) { + this.batchWriteKvCqSize = batchWriteKvCqSize; + } + + public boolean isRocksdbCQDoubleWriteEnable() { + return rocksdbCQDoubleWriteEnable; + } + + public void setRocksdbCQDoubleWriteEnable(boolean rocksdbWriteEnable) { + this.rocksdbCQDoubleWriteEnable = rocksdbWriteEnable; + } + + public boolean isTransferOffsetJsonToRocksdb() { + return transferOffsetJsonToRocksdb; + } + + public void setTransferOffsetJsonToRocksdb(boolean transferOffsetJsonToRocksdb) { + this.transferOffsetJsonToRocksdb = transferOffsetJsonToRocksdb; + } + public boolean isEnabledAppendPropCRC() { return enabledAppendPropCRC; } diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index 2f2ce981257..2401257c306 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -661,4 +661,8 @@ public void recoverTopicQueueTable() { public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { next.notifyMessageArriveIfNecessary(dispatchRequest); } + + public MessageStore getNext() { + return next; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java b/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java index b8865fd9195..34f5cb142b6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java @@ -109,6 +109,7 @@ public String toString() { ", size=" + size + ", pos=" + pos + ", batchNum=" + batchNum + + ", tagsCode=" + tagsCode + ", compactedOffset=" + compactedOffset + '}'; } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java index 5a981bb4df1..2363c2896e5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java @@ -18,7 +18,6 @@ import java.nio.ByteBuffer; import java.util.List; - import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.attribute.CQType; @@ -311,7 +310,7 @@ public CqUnit getEarliestUnit() { public CqUnit getLatestUnit() { try { long maxOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); - return get(maxOffset); + return get(maxOffset > 0 ? maxOffset - 1 : maxOffset); } catch (RocksDBException e) { ERROR_LOG.error("getLatestUnit Failed. topic: {}, queueId: {}, {}", topic, queueId, e.getMessage()); } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java index 3c6b91ec018..c889ae7ca85 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -28,7 +28,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; - import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; @@ -56,7 +55,7 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { public static final byte CTRL_1 = '\u0001'; public static final byte CTRL_2 = '\u0002'; - private static final int BATCH_SIZE = 16; + private final int batchSize; public static final int MAX_KEY_LEN = 300; private final ScheduledExecutorService scheduledExecutorService; @@ -87,10 +86,11 @@ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); this.writeBatch = new WriteBatch(); - this.bufferDRList = new ArrayList(BATCH_SIZE); - this.cqBBPairList = new ArrayList(BATCH_SIZE); - this.offsetBBPairList = new ArrayList(BATCH_SIZE); - for (int i = 0; i < BATCH_SIZE; i++) { + this.batchSize = messageStoreConfig.getBatchWriteKvCqSize(); + this.bufferDRList = new ArrayList(batchSize); + this.cqBBPairList = new ArrayList(batchSize); + this.offsetBBPairList = new ArrayList(batchSize); + for (int i = 0; i < batchSize; i++) { this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); } @@ -164,9 +164,10 @@ private boolean shutdownInner() { @Override public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { - if (request == null || this.bufferDRList.size() >= BATCH_SIZE) { + if (request == null || this.bufferDRList.size() >= batchSize) { putMessagePosition(); } + if (request != null) { this.bufferDRList.add(request); } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 1e4bbf21bd2..386cb1f6787 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -41,6 +41,7 @@ import org.apache.rocketmq.store.StoreCheckpoint; import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.Assume; import org.apache.rocketmq.common.MixAll; @@ -51,6 +52,12 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { + @BeforeClass + public static void beforeClass() { + // Temporarily skip those tests on the macOS as they are flaky + Assume.assumeFalse(MixAll.isMac()); + } + @Test public void testTruncateCQ() throws Exception { String base = createBaseDir(); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java index 5eb83207322..9de4e4820ed 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java @@ -39,6 +39,7 @@ public class DLedgerMultiPathTest extends MessageStoreTestBase { @Test public void multiDirsStorageTest() throws Exception { + Assume.assumeFalse(MixAll.isMac()); Assume.assumeFalse(MixAll.isWindows()); String base = createBaseDir(); String topic = UUID.randomUUID().toString(); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index a21806ffcf6..c4d9f0727b9 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.store.dledger; +import com.google.common.util.concurrent.RateLimiter; import io.openmessaging.storage.dledger.DLedgerConfig; import io.openmessaging.storage.dledger.DLedgerServer; import java.io.File; @@ -122,7 +123,13 @@ protected DefaultMessageStore createMessageStore(String base, boolean createAbor } protected void doPutMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) throws UnknownHostException { + RateLimiter rateLimiter = RateLimiter.create(100); + MessageStoreConfig storeConfig = messageStore.getMessageStoreConfig(); + boolean limitAppendRate = storeConfig.isEnableDLegerCommitLog(); for (int i = 0; i < num; i++) { + if (limitAppendRate) { + rateLimiter.acquire(); + } MessageExtBrokerInner msgInner = buildMessage(); msgInner.setTopic(topic); msgInner.setQueueId(queueId); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index db7b594a73b..1bfc6f72eaa 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -34,6 +34,7 @@ public class MixCommitlogTest extends MessageStoreTestBase { @Test public void testFallBehindCQ() throws Exception { Assume.assumeFalse(MixAll.isWindows()); + Assume.assumeFalse(MixAll.isMac()); String base = createBaseDir(); String topic = UUID.randomUUID().toString(); String peers = String.format("n0-localhost:%d", nextPort()); @@ -75,6 +76,7 @@ public void testFallBehindCQ() throws Exception { @Test public void testPutAndGet() throws Exception { + Assume.assumeFalse(MixAll.isMac()); String base = createBaseDir(); String topic = UUID.randomUUID().toString(); String peers = String.format("n0-localhost:%d", nextPort()); @@ -138,6 +140,7 @@ public void testPutAndGet() throws Exception { @Test public void testDeleteExpiredFiles() throws Exception { + Assume.assumeFalse(MixAll.isMac()); String base = createBaseDir(); String topic = UUID.randomUUID().toString(); String peers = String.format("n0-localhost:%d", nextPort()); diff --git a/test/pom.xml b/test/pom.xml index df380a0b604..801a10301eb 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java index b64cda33420..472e106ce35 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java @@ -100,8 +100,8 @@ public class BaseConf { brokerController2.getBrokerConfig().getListenPort()); brokerController3 = IntegrationTestBase.createAndStartBroker(NAMESRV_ADDR); - log.debug("Broker {} started, listening: {}", brokerController2.getBrokerConfig().getBrokerName(), - brokerController2.getBrokerConfig().getListenPort()); + log.debug("Broker {} started, listening: {}", brokerController3.getBrokerConfig().getBrokerName(), + brokerController3.getBrokerConfig().getListenPort()); CLUSTER_NAME = brokerController1.getBrokerConfig().getBrokerClusterName(); BROKER1_NAME = brokerController1.getBrokerConfig().getBrokerName(); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java index 684b718ae5d..7408a092c4b 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java @@ -96,6 +96,8 @@ public void test3ConsumerAndCrashOne() { MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(), consumer2.getListener(), consumer3.getListener()); consumer3.shutdown(); + TestUtils.waitForSeconds(WAIT_TIME); + producer.clearMsg(); consumer1.clearMsg(); consumer2.clearMsg(); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java index b5d79d6c0ae..3a6ad060020 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java @@ -30,6 +30,7 @@ import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; +import org.junit.Ignore; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -53,6 +54,7 @@ public void setUp() { } @Test + @Ignore public void testNotification() throws Exception { long pollTime = 500; CompletableFuture future1 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), pollTime, System.currentTimeMillis(), 5000); diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java index 33c3aa2fb89..77f5f362125 100644 --- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java @@ -30,6 +30,7 @@ import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; +import org.junit.Ignore; import org.junit.runners.MethodSorters; import static org.awaitility.Awaitility.await; @@ -87,6 +88,7 @@ public void testTransactionCheckThenCommit() { } @Test + @Ignore public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { super.testSimpleConsumerSendAndRecvDelayMessage(); } diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java index 7f837adebe5..515c3f121dd 100644 --- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java @@ -26,6 +26,7 @@ import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; +import org.junit.Ignore; import org.junit.runners.MethodSorters; @FixMethodOrder(value = MethodSorters.NAME_ASCENDING) @@ -75,6 +76,7 @@ public void testTransactionCheckThenCommit() { } @Test + @Ignore public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { super.testSimpleConsumerSendAndRecvDelayMessage(); } diff --git a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java index 9004b91db39..9e9afb1ed2c 100644 --- a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java @@ -17,13 +17,16 @@ package org.apache.rocketmq.test.route; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; import org.apache.rocketmq.test.base.BaseConf; import org.apache.rocketmq.test.util.MQAdminTestUtils; +import org.junit.Ignore; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; public class CreateAndUpdateTopicIT extends BaseConf { @@ -47,6 +50,8 @@ public void testCreateOrUpdateTopic_EnableSingleTopicRegistration() { } + // Temporarily ignore the fact that this test cannot pass in the integration test pipeline due to unknown reasons + @Ignore @Test public void testDeleteTopicFromNameSrvWithBrokerRegistration() { namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(true); @@ -60,11 +65,9 @@ public void testDeleteTopicFromNameSrvWithBrokerRegistration() { boolean createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic1, 8, null); assertThat(createResult).isTrue(); - createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic2, 8, null); assertThat(createResult).isTrue(); - TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2); assertThat(route.getBrokerDatas()).hasSize(3); @@ -73,11 +76,13 @@ public void testDeleteTopicFromNameSrvWithBrokerRegistration() { // Deletion is lazy, trigger broker registration brokerController1.registerBrokerAll(false, false, true); - // The route info of testTopic2 will be removed from broker1 after the registration - route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2); - assertThat(route.getBrokerDatas()).hasSize(2); - assertThat(route.getQueueDatas().get(0).getBrokerName()).isEqualTo(BROKER2_NAME); - assertThat(route.getQueueDatas().get(1).getBrokerName()).isEqualTo(BROKER3_NAME); + await().atMost(10, TimeUnit.SECONDS).until(() -> { + // The route info of testTopic2 will be removed from broker1 after the registration + TopicRouteData finalRoute = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2); + return finalRoute.getBrokerDatas().size() == 2 + && finalRoute.getQueueDatas().get(0).getBrokerName().equals(BROKER2_NAME) + && finalRoute.getQueueDatas().get(1).getBrokerName().equals(BROKER3_NAME); + }); brokerController1.getBrokerConfig().setEnableSingleTopicRegister(false); brokerController2.getBrokerConfig().setEnableSingleTopicRegister(false); diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml index 96f042da21b..4d9af208187 100644 --- a/tieredstore/pom.xml +++ b/tieredstore/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java index 7b63e16696e..0e3ede871c3 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java @@ -180,9 +180,15 @@ public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int } // determine whether tiered storage path conditions are met - if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_DISK) - && !next.checkInStoreByConsumeOffset(topic, queueId, offset)) { - return true; + if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_DISK)) { + // return true to read from tiered storage if the CommitLog is empty + if (next != null && next.getCommitLog() != null && + next.getCommitLog().getMinOffset() < 0L) { + return true; + } + if (!next.checkInStoreByConsumeOffset(topic, queueId, offset)) { + return true; + } } if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_MEM) @@ -208,10 +214,10 @@ public CompletableFuture getMessageAsync(String group, String } if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { - log.trace("GetMessageAsync from current store, " + + log.trace("GetMessageAsync from remote store, " + "topic: {}, queue: {}, offset: {}, maxCount: {}", topic, queueId, offset, maxMsgNums); } else { - log.trace("GetMessageAsync from remote store, " + + log.trace("GetMessageAsync from next store, " + "topic: {}, queue: {}, offset: {}, maxCount: {}", topic, queueId, offset, maxMsgNums); return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java index 020b9f3b068..0db5dc5c4c5 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java @@ -42,8 +42,6 @@ import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.tieredstore.MessageStoreConfig; import org.apache.rocketmq.tieredstore.common.AppendResult; -import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode; -import org.apache.rocketmq.tieredstore.exception.TieredStoreException; import org.apache.rocketmq.tieredstore.file.FlatAppendFile; import org.apache.rocketmq.tieredstore.file.FlatFileFactory; import org.apache.rocketmq.tieredstore.provider.FileSegment; @@ -271,23 +269,23 @@ public CompletableFuture> queryAsync( public void forceUpload() { try { readWriteLock.writeLock().lock(); - if (this.currentWriteFile == null) { - log.warn("IndexStoreService no need force upload current write file"); - return; - } - // note: current file has been shutdown before - IndexStoreFile lastFile = new IndexStoreFile(storeConfig, currentWriteFile.getTimestamp()); - if (this.doCompactThenUploadFile(lastFile)) { - this.setCompactTimestamp(lastFile.getTimestamp()); - } else { - throw new TieredStoreException( - TieredStoreErrorCode.UNKNOWN, "IndexStoreService force compact current file error"); + while (true) { + Map.Entry entry = + this.timeStoreTable.higherEntry(this.compactTimestamp.get()); + if (entry == null) { + break; + } + if (this.doCompactThenUploadFile(entry.getValue())) { + this.setCompactTimestamp(entry.getValue().getTimestamp()); + // The total number of files will not too much, prevent io too fast. + TimeUnit.MILLISECONDS.sleep(50); + } } } catch (Exception e) { log.error("IndexStoreService force upload error", e); throw new RuntimeException(e); } finally { - readWriteLock.writeLock().lock(); + readWriteLock.writeLock().unlock(); } } @@ -393,19 +391,13 @@ protected IndexFile getNextSealedFile() { @Override public void shutdown() { super.shutdown(); - readWriteLock.writeLock().lock(); - try { - for (Map.Entry entry : timeStoreTable.entrySet()) { - entry.getValue().shutdown(); - } - if (!autoCreateNewFile) { - this.forceUpload(); + // Wait index service upload then clear time store table + while (!this.timeStoreTable.isEmpty()) { + try { + TimeUnit.MILLISECONDS.sleep(50); + } catch (InterruptedException e) { + throw new RuntimeException(e); } - this.timeStoreTable.clear(); - } catch (Exception e) { - log.error("IndexStoreService shutdown error", e); - } finally { - readWriteLock.writeLock().unlock(); } } @@ -424,6 +416,18 @@ public void run() { } this.waitForRunning(TimeUnit.SECONDS.toMillis(10)); } + readWriteLock.writeLock().lock(); + try { + if (autoCreateNewFile) { + this.forceUpload(); + } + this.timeStoreTable.forEach((timestamp, file) -> file.shutdown()); + this.timeStoreTable.clear(); + } catch (Exception e) { + log.error("IndexStoreService shutdown error", e); + } finally { + readWriteLock.writeLock().unlock(); + } log.info(this.getServiceName() + " service shutdown"); } } diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java index fb563f7c6c2..83b407e73ba 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java @@ -120,7 +120,7 @@ public void doConvertOldFormatTest() throws IOException { indexService = new IndexStoreService(fileAllocator, filePath); indexService.start(); ConcurrentSkipListMap timeStoreTable = indexService.getTimeStoreTable(); - Assert.assertEquals(1, timeStoreTable.size()); + Assert.assertEquals(2, timeStoreTable.size()); Assert.assertEquals(Long.valueOf(timestamp), timeStoreTable.firstKey()); mappedFile.destroy(10 * 1000); } @@ -232,7 +232,7 @@ public void restartServiceTest() throws InterruptedException { indexService = new IndexStoreService(fileAllocator, filePath); indexService.start(); Assert.assertEquals(timestamp, indexService.getTimeStoreTable().firstKey().longValue()); - Assert.assertEquals(2, indexService.getTimeStoreTable().size()); + Assert.assertEquals(4, indexService.getTimeStoreTable().size()); Assert.assertEquals(IndexFile.IndexStatusEnum.UPLOAD, indexService.getTimeStoreTable().firstEntry().getValue().getFileStatus()); } diff --git a/tools/pom.xml b/tools/pom.xml index ee459dfd95a..ab740bd8a70 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.2-SNAPSHOT 4.0.0 diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 5be6d24ff76..3686bf2644b 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -52,6 +52,7 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -153,6 +154,12 @@ public QueryResult queryMessage(String topic, String key, int maxNum, long begin return defaultMQAdminExtImpl.queryMessage(topic, key, maxNum, begin, end); } + + public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException, RemotingException { + return defaultMQAdminExtImpl.queryMessage(clusterName, topic, key, maxNum, begin, end); + } + @Override public void start() throws MQClientException { defaultMQAdminExtImpl.start(); @@ -196,7 +203,8 @@ public void createAndUpdateTopicConfig(String addr, TopicConfig config) throws R } @Override - public void createAndUpdateTopicConfigList(String addr, List topicConfigList) throws InterruptedException, RemotingException, MQClientException { + public void createAndUpdateTopicConfigList(String addr, + List topicConfigList) throws InterruptedException, RemotingException, MQClientException { defaultMQAdminExtImpl.createAndUpdateTopicConfigList(addr, topicConfigList); } @@ -300,6 +308,12 @@ public ConsumeStats examineConsumeStats( return examineConsumeStats(consumerGroup, null); } + @Override + public ConsumeStats examineConsumeStats(String clusterName, String consumerGroup, + String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.examineConsumeStats(clusterName, consumerGroup, topic); + } + @Override public ConsumeStats examineConsumeStats(String consumerGroup, String topic) throws RemotingException, MQClientException, @@ -459,16 +473,35 @@ public List resetOffsetByTimestampOld(String consumerGroup, Strin return defaultMQAdminExtImpl.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force); } + public List resetOffsetByTimestampOld(String clusterName, String consumerGroup, String topic, long timestamp, + boolean force) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestampOld(clusterName, consumerGroup, topic, timestamp, force); + } + + @Override + public Map resetOffsetByTimestamp(String clusterName, String topic, String group, + long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce); + } + @Override public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { return resetOffsetByTimestamp(topic, group, timestamp, isForce, false); } + public Map resetOffsetByTimestamp(String clusterName, String topic, String group, + long timestamp, boolean isForce, boolean isC) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce, isC); + } + + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce, boolean isC) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - return defaultMQAdminExtImpl.resetOffsetByTimestamp(topic, group, timestamp, isForce, isC); + return defaultMQAdminExtImpl.resetOffsetByTimestamp(null, topic, group, timestamp, isForce, isC); } @Override @@ -589,10 +622,19 @@ public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String c @Override public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId, - final String topic, final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + final String topic, + final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { return defaultMQAdminExtImpl.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); } + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(final String clusterName, final String consumerGroup, + final String clientId, + final String topic, + final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.consumeMessageDirectly(clusterName, consumerGroup, clientId, topic, msgId); + } + @Override public List messageTrackDetail( MessageExt msg) throws RemotingException, MQClientException, InterruptedException, @@ -730,6 +772,12 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String ); } + @Override + public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { + return this.defaultMQAdminExtImpl.checkRocksdbCqWriteProgress(brokerAddr, topic); + } + @Override public boolean resumeCheckHalfMessage(String topic, String msgId) @@ -796,10 +844,10 @@ public void resetMasterFlushOffset(String brokerAddr, long masterFlushOffset) this.defaultMQAdminExtImpl.resetMasterFlushOffset(brokerAddr, masterFlushOffset); } - public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, long end) + public QueryResult queryMessageByUniqKey(String clusterName, String topic, String key, int maxNum, long begin, + long end) throws MQClientException, InterruptedException { - - return defaultMQAdminExtImpl.queryMessageByUniqKey(topic, key, maxNum, begin, end); + return defaultMQAdminExtImpl.queryMessageByUniqKey(clusterName, topic, key, maxNum, begin, end); } public DefaultMQAdminExtImpl getDefaultMQAdminExtImpl() { @@ -831,13 +879,14 @@ public void updateControllerConfig(Properties properties, @Override public Pair electMaster(String controllerAddr, String clusterName, - String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException { + String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException { return this.defaultMQAdminExtImpl.electMaster(controllerAddr, clusterName, brokerName, brokerId); } @Override public void cleanControllerBrokerData(String controllerAddr, String clusterName, String brokerName, - String brokerControllerIdsToClean, boolean isCleanLivingBroker) throws RemotingException, InterruptedException, MQBrokerException { + String brokerControllerIdsToClean, + boolean isCleanLivingBroker) throws RemotingException, InterruptedException, MQBrokerException { this.defaultMQAdminExtImpl.cleanControllerBrokerData(controllerAddr, clusterName, brokerName, brokerControllerIdsToClean, isCleanLivingBroker); } @@ -876,13 +925,15 @@ public void createUser(String brokerAddr, } @Override - public void createUser(String brokerAddr, String username, String password, String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void createUser(String brokerAddr, String username, String password, + String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.createUser(brokerAddr, username, password, userType); } @Override public void updateUser(String brokerAddr, String username, - String password, String userType, String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + String password, String userType, + String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.updateUser(brokerAddr, username, password, userType, userStatus); } @@ -912,38 +963,45 @@ public List listUser(String brokerAddr, @Override public void createAcl(String brokerAddr, String subject, List resources, List actions, - List sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + List sourceIps, + String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.createAcl(brokerAddr, subject, resources, actions, sourceIps, decision); } @Override - public void createAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void createAcl(String brokerAddr, + AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.createAcl(brokerAddr, aclInfo); } @Override public void updateAcl(String brokerAddr, String subject, List resources, List actions, - List sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + List sourceIps, + String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.updateAcl(brokerAddr, subject, resources, actions, sourceIps, decision); } @Override - public void updateAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void updateAcl(String brokerAddr, + AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.updateAcl(brokerAddr, aclInfo); } @Override - public void deleteAcl(String brokerAddr, String subject, String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void deleteAcl(String brokerAddr, String subject, + String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.deleteAcl(brokerAddr, subject, resource); } @Override - public AclInfo getAcl(String brokerAddr, String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public AclInfo getAcl(String brokerAddr, + String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { return defaultMQAdminExtImpl.getAcl(brokerAddr, subject); } @Override - public List listAcl(String brokerAddr, String subjectFilter, String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public List listAcl(String brokerAddr, String subjectFilter, + String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { return defaultMQAdminExtImpl.listAcl(brokerAddr, subjectFilter, resourceFilter); } } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 9546235d3e8..883dcbe41d7 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -90,6 +90,7 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -422,14 +423,21 @@ public KVTable fetchBrokerRuntimeStats( return this.mqClientInstance.getMQClientAPIImpl().getBrokerRuntimeInfo(brokerAddr, timeoutMillis); } + @Override + public ConsumeStats examineConsumeStats( + String consumerGroup, + String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return examineConsumeStats(null, consumerGroup, topic); + } + @Override public ConsumeStats examineConsumeStats( String consumerGroup) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { - return examineConsumeStats(consumerGroup, null); + return examineConsumeStats(null, consumerGroup, null); } @Override - public ConsumeStats examineConsumeStats(String consumerGroup, + public ConsumeStats examineConsumeStats(String clusterName, String consumerGroup, String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { TopicRouteData topicRouteData = null; List routeTopics = new ArrayList<>(); @@ -438,6 +446,12 @@ public ConsumeStats examineConsumeStats(String consumerGroup, routeTopics.add(topic); routeTopics.add(KeyBuilder.buildPopRetryTopic(topic, consumerGroup)); } + + // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer + if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) && !StringUtils.isEmpty(clusterName)) { + routeTopics.add(clusterName); + } + for (int i = 0; i < routeTopics.size(); i++) { try { topicRouteData = this.examineTopicRouteInfo(routeTopics.get(i)); @@ -467,25 +481,33 @@ public ConsumeStats examineConsumeStats(String consumerGroup, topics.add(messageQueue.getTopic()); } - ConsumeStats staticResult = new ConsumeStats(); - staticResult.setConsumeTps(result.getConsumeTps()); - // for topic, we put the physical stats, how about group? - // staticResult.getOffsetTable().putAll(result.getOffsetTable()); - - for (String currentTopic : topics) { - TopicRouteData currentRoute = this.examineTopicRouteInfo(currentTopic); - if (currentRoute.getTopicQueueMappingByBroker() == null - || currentRoute.getTopicQueueMappingByBroker().isEmpty()) { - //normal topic - for (Map.Entry entry : result.getOffsetTable().entrySet()) { - if (entry.getKey().getTopic().equals(currentTopic)) { - staticResult.getOffsetTable().put(entry.getKey(), entry.getValue()); + ConsumeStats staticResult = null; + + if (StringUtils.isEmpty(clusterName)) { + + staticResult = new ConsumeStats(); + staticResult.setConsumeTps(result.getConsumeTps()); + // for topic, we put the physical stats, how about group? + // staticResult.getOffsetTable().putAll(result.getOffsetTable()); + + for (String currentTopic : topics) { + TopicRouteData currentRoute = this.examineTopicRouteInfo(currentTopic); + if (currentRoute.getTopicQueueMappingByBroker() == null + || currentRoute.getTopicQueueMappingByBroker().isEmpty()) { + //normal topic + for (Map.Entry entry : result.getOffsetTable().entrySet()) { + if (entry.getKey().getTopic().equals(currentTopic)) { + staticResult.getOffsetTable().put(entry.getKey(), entry.getValue()); + } } } + Map brokerConfigMap = MQAdminUtils.examineTopicConfigFromRoute(currentTopic, currentRoute, defaultMQAdminExt); + ConsumeStats consumeStats = MQAdminUtils.convertPhysicalConsumeStats(brokerConfigMap, result); + staticResult.getOffsetTable().putAll(consumeStats.getOffsetTable()); } - Map brokerConfigMap = MQAdminUtils.examineTopicConfigFromRoute(currentTopic, currentRoute, defaultMQAdminExt); - ConsumeStats consumeStats = MQAdminUtils.convertPhysicalConsumeStats(brokerConfigMap, result); - staticResult.getOffsetTable().putAll(consumeStats.getOffsetTable()); + + } else { + staticResult = result; } if (staticResult.getOffsetTable().isEmpty()) { @@ -811,10 +833,16 @@ public void deleteKvConfig(String namespace, this.mqClientInstance.getMQClientAPIImpl().deleteKVConfigValue(namespace, key, timeoutMillis); } - @Override - public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, + public List resetOffsetByTimestampOld(String clusterName, String consumerGroup, String topic, + long timestamp, boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + String routeTopic = topic; + // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer + if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) + && !StringUtils.isEmpty(clusterName)) { + routeTopic = clusterName; + } + TopicRouteData topicRouteData = this.examineTopicRouteInfo(routeTopic); List rollbackStatsList = new ArrayList<>(); Map topicRouteMap = new HashMap<>(); for (QueueData queueData : topicRouteData.getQueueDatas()) { @@ -829,6 +857,12 @@ public List resetOffsetByTimestampOld(String consumerGroup, Strin return rollbackStatsList; } + @Override + public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, + boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return resetOffsetByTimestampOld(null, consumerGroup, topic, timestamp, force); + } + private List resetOffsetByTimestampOld(String brokerAddr, QueueData queueData, String consumerGroup, String topic, long timestamp, boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { @@ -864,7 +898,7 @@ private List resetOffsetByTimestampOld(String brokerAddr, QueueDa @Override public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - return resetOffsetByTimestamp(topic, group, timestamp, isForce, false); + return resetOffsetByTimestamp(null, topic, group, timestamp, isForce, false); } @Override @@ -951,9 +985,16 @@ public void run() { }); } - public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce, + public Map resetOffsetByTimestamp(String clusterName, String topic, String group, + long timestamp, boolean isForce, boolean isC) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + String routeTopic = topic; + // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer + if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) + && !StringUtils.isEmpty(clusterName)) { + routeTopic = clusterName; + } + TopicRouteData topicRouteData = this.examineTopicRouteInfo(routeTopic); List brokerDatas = topicRouteData.getBrokerDatas(); Map allOffsetTable = new HashMap<>(); if (brokerDatas != null) { @@ -1325,7 +1366,8 @@ public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String c @Override public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId, - final String topic, final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + final String topic, + final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { MessageExt msg = this.viewMessage(topic, msgId); if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) { return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgId, timeoutMillis); @@ -1335,6 +1377,20 @@ public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumer } } + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(final String clusterName, final String consumerGroup, + final String clientId, + final String topic, + final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + MessageExt msg = this.queryMessage(clusterName, topic, msgId); + if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) { + return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgId, timeoutMillis); + } else { + MessageClientExt msgClient = (MessageClientExt) msg; + return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgClient.getOffsetMsgId(), timeoutMillis); + } + } + @Override public List messageTrackDetail( MessageExt msg) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { @@ -1664,10 +1720,10 @@ public TopicConfigSerializeWrapper getUserTopicConfig(final String brokerAddr, f while (iterator.hasNext()) { TopicConfig topicConfig = iterator.next().getValue(); if (topicList.getTopicList().contains(topicConfig.getTopicName()) - || TopicValidator.isSystemTopic(topicConfig.getTopicName())) { + || TopicValidator.isSystemTopic(topicConfig.getTopicName())) { iterator.remove(); } else if (!specialTopic && StringUtils.startsWithAny(topicConfig.getTopicName(), - MixAll.RETRY_GROUP_TOPIC_PREFIX, MixAll.DLQ_GROUP_TOPIC_PREFIX)) { + MixAll.RETRY_GROUP_TOPIC_PREFIX, MixAll.DLQ_GROUP_TOPIC_PREFIX)) { iterator.remove(); } else if (!PermName.isValid(topicConfig.getPerm())) { iterator.remove(); @@ -1726,6 +1782,11 @@ public QueryResult queryMessage(String topic, String key, int maxNum, long begin return this.mqClientInstance.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); } + public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, + long end) throws MQClientException, InterruptedException, RemotingException { + return this.mqClientInstance.getMQAdminImpl().queryMessage(clusterName, topic, key, maxNum, begin, end, false); + } + @Override public void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq, long offset) throws RemotingException, InterruptedException, MQBrokerException { @@ -1757,6 +1818,12 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String return this.mqClientInstance.getMQClientAPIImpl().queryConsumeQueue(brokerAddr, topic, queueId, index, count, consumerGroup, timeoutMillis); } + @Override + public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { + return this.mqClientInstance.getMQClientAPIImpl().checkRocksdbCqWriteProgress(brokerAddr, topic, timeoutMillis); + } + @Override public boolean resumeCheckHalfMessage(final String topic, final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { @@ -1783,10 +1850,9 @@ public long searchOffset(final String brokerAddr, final String topicName, final return this.mqClientInstance.getMQClientAPIImpl().searchOffset(brokerAddr, topicName, queueId, timestamp, timeoutMillis); } - public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, + public QueryResult queryMessageByUniqKey(String clusterName, String topic, String key, int maxNum, long begin, long end) throws MQClientException, InterruptedException { - - return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, key, maxNum, begin, end); + return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(clusterName, topic, key, maxNum, begin, end); } @Override @@ -1812,6 +1878,12 @@ public void resetOffsetByQueueId(final String brokerAddr, final String consumeGr } } + @Override + public Map resetOffsetByTimestamp(String clusterName, String topic, String group, + long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce, false); + } + @Override public HARuntimeInfo getBrokerHAStatus( String brokerAddr) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { @@ -1844,7 +1916,7 @@ public void resetMasterFlushOffset(String brokerAddr, @Override public Pair electMaster(String controllerAddr, String clusterName, - String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException { + String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException { return this.mqClientInstance.getMQClientAPIImpl().electMaster(controllerAddr, clusterName, brokerName, brokerId); } @@ -1930,20 +2002,23 @@ public void createUser(String brokerAddr, } @Override - public void createUser(String brokerAddr, String username, String password, String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void createUser(String brokerAddr, String username, String password, + String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { UserInfo userInfo = UserInfo.of(username, password, userType); this.createUser(brokerAddr, userInfo); } @Override public void updateUser(String brokerAddr, String username, - String password, String userType, String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + String password, String userType, + String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { UserInfo userInfo = UserInfo.of(username, password, userType, userStatus); this.mqClientInstance.getMQClientAPIImpl().updateUser(brokerAddr, userInfo, timeoutMillis); } @Override - public void updateUser(String brokerAddr, UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void updateUser(String brokerAddr, + UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { this.mqClientInstance.getMQClientAPIImpl().updateUser(brokerAddr, userInfo, timeoutMillis); } @@ -1967,40 +2042,47 @@ public List listUser(String brokerAddr, @Override public void createAcl(String brokerAddr, String subject, List resources, List actions, - List sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + List sourceIps, + String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { AclInfo aclInfo = AclInfo.of(subject, resources, actions, sourceIps, decision); this.createAcl(brokerAddr, aclInfo); } @Override - public void createAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void createAcl(String brokerAddr, + AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { this.mqClientInstance.getMQClientAPIImpl().createAcl(brokerAddr, aclInfo, timeoutMillis); } @Override public void updateAcl(String brokerAddr, String subject, List resources, List actions, - List sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + List sourceIps, + String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { AclInfo aclInfo = AclInfo.of(subject, resources, actions, sourceIps, decision); this.updateAcl(brokerAddr, aclInfo); } @Override - public void updateAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void updateAcl(String brokerAddr, + AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { this.mqClientInstance.getMQClientAPIImpl().updateAcl(brokerAddr, aclInfo, timeoutMillis); } @Override - public void deleteAcl(String brokerAddr, String subject, String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void deleteAcl(String brokerAddr, String subject, + String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { this.mqClientInstance.getMQClientAPIImpl().deleteAcl(brokerAddr, subject, resource, timeoutMillis); } @Override - public AclInfo getAcl(String brokerAddr, String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public AclInfo getAcl(String brokerAddr, + String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { return this.mqClientInstance.getMQClientAPIImpl().getAcl(brokerAddr, subject, timeoutMillis); } @Override - public List listAcl(String brokerAddr, String subjectFilter, String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public List listAcl(String brokerAddr, String subjectFilter, + String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { return this.mqClientInstance.getMQClientAPIImpl().listAcl(brokerAddr, subjectFilter, resourceFilter, timeoutMillis); } } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index 9dff3cbab95..09204ab7be2 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -148,10 +149,16 @@ ConsumeStats examineConsumeStats( final String consumerGroup) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException; + ConsumeStats examineConsumeStats(final String consumerGroup, final String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + ConsumeStats examineConsumeStats(final String clusterName, final String consumerGroup, + final String topic) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException; + ConsumeStats examineConsumeStats(final String brokerAddr, final String consumerGroup, final String topicName, final long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException; @@ -232,6 +239,9 @@ List resetOffsetByTimestampOld(String consumerGroup, String topic Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + Map resetOffsetByTimestamp(String clusterName, String topic, String group, long timestamp, boolean isForce) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; @@ -293,6 +303,11 @@ ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, String topic, String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + ConsumeMessageDirectlyResult consumeMessageDirectly(String clusterName, String consumerGroup, + String clientId, + String topic, + String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + List messageTrackDetail( MessageExt msg) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index 43e4259c4e1..313a777ce4f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -104,6 +104,7 @@ import org.apache.rocketmq.tools.command.offset.ResetOffsetByTimeCommand; import org.apache.rocketmq.tools.command.offset.SkipAccumulationSubCommand; import org.apache.rocketmq.tools.command.producer.ProducerSubCommand; +import org.apache.rocketmq.tools.command.queue.CheckRocksdbCqWriteProgressCommand; import org.apache.rocketmq.tools.command.queue.QueryConsumeQueueCommand; import org.apache.rocketmq.tools.command.stats.StatsAllSubCommand; import org.apache.rocketmq.tools.command.topic.AllocateMQSubCommand; @@ -304,6 +305,7 @@ public static void initCommand() { initCommand(new ListAclSubCommand()); initCommand(new CopyAclsSubCommand()); initCommand(new RocksDBConfigToJsonCommand()); + initCommand(new CheckRocksdbCqWriteProgressCommand()); } private static void printHelp() { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java index c489cad6849..b638dcf61f3 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -72,6 +72,10 @@ public Options buildCommandlineOptions(Options options) { optionShowClientIP.setRequired(false); options.addOption(optionShowClientIP); + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -109,6 +113,8 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t boolean showClientIP = commandLine.hasOption('s') && "true".equalsIgnoreCase(commandLine.getOptionValue('s')); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; + if (commandLine.hasOption('g')) { String consumerGroup = commandLine.getOptionValue('g').trim(); String topicName = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : null; @@ -116,7 +122,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t if (topicName == null) { consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup); } else { - consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup, topicName); + consumeStats = defaultMQAdminExt.examineConsumeStats(clusterName, consumerGroup, topicName); } List mqList = new LinkedList<>(consumeStats.getOffsetTable().keySet()); Collections.sort(mqList); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java index 1ecb1fa2cd9..c466490b8a8 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.rocketmq.tools.command.export; import com.alibaba.fastjson.JSONObject; @@ -77,6 +78,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + path += "/" + configType; boolean jsonEnable = false; if (commandLine.hasOption("jsonEnable")) { @@ -86,7 +88,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t ConfigRocksDBStorage kvStore = new ConfigRocksDBStorage(path, true /* readOnly */); if (!kvStore.start()) { - System.out.print("RocksDB load error, path=" + path + "\n"); + System.out.printf("RocksDB load error, path=%s\n" , path); return; } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java index 5245ca089ff..e83029eed31 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java @@ -44,9 +44,10 @@ import org.apache.rocketmq.tools.command.SubCommandException; public class QueryMsgByIdSubCommand implements SubCommand { - public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId, final Charset msgBodyCharset) throws MQClientException, + public static void queryById(final DefaultMQAdminExt admin, final String clusterName, final String topic, + final String msgId, final Charset msgBodyCharset) throws MQClientException, RemotingException, MQBrokerException, InterruptedException, IOException { - MessageExt msg = admin.viewMessage(topic, msgId); + MessageExt msg = admin.queryMessage(clusterName, topic, msgId); printMsg(admin, msg, msgBodyCharset); } @@ -55,7 +56,8 @@ public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg) printMsg(admin, msg, null); } - public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg, final Charset msgBodyCharset) throws IOException { + public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg, + final Charset msgBodyCharset) throws IOException { if (msg == null) { System.out.printf("%nMessage not found!"); return; @@ -219,6 +221,10 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -244,13 +250,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t final String msgIds = commandLine.getOptionValue('i').trim(); final String[] msgIdArr = StringUtils.split(msgIds, ","); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; if (commandLine.hasOption('g') && commandLine.hasOption('d')) { final String consumerGroup = commandLine.getOptionValue('g').trim(); final String clientId = commandLine.getOptionValue('d').trim(); for (String msgId : msgIdArr) { if (StringUtils.isNotBlank(msgId)) { - pushMsg(defaultMQAdminExt, consumerGroup, clientId, topic, msgId.trim()); + pushMsg(defaultMQAdminExt, clusterName, consumerGroup, clientId, topic, msgId.trim()); } } } else if (commandLine.hasOption('s')) { @@ -258,7 +265,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t if (resend) { for (String msgId : msgIdArr) { if (StringUtils.isNotBlank(msgId)) { - sendMsg(defaultMQAdminExt, defaultMQProducer, topic, msgId.trim()); + sendMsg(defaultMQAdminExt, clusterName, defaultMQProducer, topic, msgId.trim()); } } } @@ -269,7 +276,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } for (String msgId : msgIdArr) { if (StringUtils.isNotBlank(msgId)) { - queryById(defaultMQAdminExt, topic, msgId.trim(), msgBodyCharset); + queryById(defaultMQAdminExt, clusterName, topic, msgId.trim(), msgBodyCharset); } } @@ -282,13 +289,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } } - private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String consumerGroup, final String clientId, + private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String clusterName, + final String consumerGroup, final String clientId, final String topic, final String msgId) { try { ConsumerRunningInfo consumerRunningInfo = defaultMQAdminExt.getConsumerRunningInfo(consumerGroup, clientId, false, false); if (consumerRunningInfo != null && ConsumerRunningInfo.isPushType(consumerRunningInfo)) { ConsumeMessageDirectlyResult result = - defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); + defaultMQAdminExt.consumeMessageDirectly(clusterName, consumerGroup, clientId, topic, msgId); System.out.printf("%s", result); } else { System.out.printf("this %s client is not push consumer ,not support direct push \n", clientId); @@ -298,10 +306,11 @@ private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String con } } - private void sendMsg(final DefaultMQAdminExt defaultMQAdminExt, final DefaultMQProducer defaultMQProducer, + private void sendMsg(final DefaultMQAdminExt defaultMQAdminExt, final String clusterName, + final DefaultMQProducer defaultMQProducer, final String topic, final String msgId) { try { - MessageExt msg = defaultMQAdminExt.viewMessage(topic, msgId); + MessageExt msg = defaultMQAdminExt.queryMessage(clusterName, topic, msgId); if (msg != null) { // resend msg by id System.out.printf("prepare resend msg. originalMsgId=%s", msgId); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java index 64627fd19fa..02961c3bb50 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java @@ -23,6 +23,8 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; + import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; @@ -41,7 +43,7 @@ public String commandDesc() { @Override public Options buildCommandlineOptions(Options options) { - Option opt = new Option("t", "topic", true, "topic name"); + Option opt = new Option("t", "topic", true, "Topic name"); opt.setRequired(true); options.addOption(opt); @@ -57,7 +59,11 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); - opt = new Option("c", "maxNum", true, "The maximum number of messages returned by the query, default:64"); + opt = new Option("m", "maxNum", true, "The maximum number of messages returned by the query, default:64"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); opt.setRequired(false); options.addOption(opt); @@ -77,16 +83,20 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t long beginTimestamp = 0; long endTimestamp = Long.MAX_VALUE; int maxNum = 64; + String clusterName = null; if (commandLine.hasOption("b")) { beginTimestamp = Long.parseLong(commandLine.getOptionValue("b").trim()); } if (commandLine.hasOption("e")) { endTimestamp = Long.parseLong(commandLine.getOptionValue("e").trim()); } + if (commandLine.hasOption("m")) { + maxNum = Integer.parseInt(commandLine.getOptionValue("m").trim()); + } if (commandLine.hasOption("c")) { - maxNum = Integer.parseInt(commandLine.getOptionValue("c").trim()); + clusterName = commandLine.getOptionValue("c").trim(); } - this.queryByKey(defaultMQAdminExt, topic, key, maxNum, beginTimestamp, endTimestamp); + this.queryByKey(defaultMQAdminExt, clusterName, topic, key, maxNum, beginTimestamp, endTimestamp); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); } finally { @@ -94,12 +104,13 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } } - private void queryByKey(final DefaultMQAdminExt admin, final String topic, final String key, int maxNum, long begin, + private void queryByKey(final DefaultMQAdminExt admin, final String cluster, final String topic, final String key, int maxNum, long begin, long end) - throws MQClientException, InterruptedException { + throws MQClientException, InterruptedException, RemotingException { admin.start(); - QueryResult queryResult = admin.queryMessage(topic, key, maxNum, begin, end); + QueryResult queryResult = admin.queryMessage(cluster, topic, key, maxNum, begin, end); + System.out.printf("%-50s %4s %40s%n", "#Message ID", "#QID", diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java index b71cee90160..5295d91cc30 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java @@ -25,13 +25,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.rocketmq.client.QueryResult; -import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; -import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; @@ -51,19 +49,18 @@ private DefaultMQAdminExt createMQAdminExt(RPCHook rpcHook) throws SubCommandExc defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); try { defaultMQAdminExt.start(); - } - catch (Exception e) { + } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); } return defaultMQAdminExt; } } - public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId, - final boolean showAll) throws MQClientException, - RemotingException, MQBrokerException, InterruptedException, IOException { + public static void queryById(final DefaultMQAdminExt admin, final String clusterName, final String topic, + final String msgId, + final boolean showAll) throws MQClientException, InterruptedException, IOException { - QueryResult queryResult = admin.queryMessageByUniqKey(topic, msgId, 32, 0, Long.MAX_VALUE); + QueryResult queryResult = admin.queryMessageByUniqKey(clusterName, topic, msgId, 32, 0, Long.MAX_VALUE); assert queryResult != null; List list = queryResult.getMessageList(); if (list == null || list.size() == 0) { @@ -94,7 +91,7 @@ private static void showMessage(final DefaultMQAdminExt admin, MessageExt msg, i System.out.printf(strFormat, "Store Host:", RemotingHelper.parseSocketAddressAddr(msg.getStoreHost())); System.out.printf(intFormat, "System Flag:", msg.getSysFlag()); System.out.printf(strFormat, "Properties:", - msg.getProperties() != null ? msg.getProperties().toString() : ""); + msg.getProperties() != null ? msg.getProperties().toString() : ""); System.out.printf(strFormat, "Message Body Path:", bodyTmpFilePath); try { @@ -166,6 +163,10 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -173,10 +174,11 @@ public Options buildCommandlineOptions(Options options) { public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { try { - defaultMQAdminExt = createMQAdminExt(rpcHook); + defaultMQAdminExt = createMQAdminExt(rpcHook); final String msgId = commandLine.getOptionValue('i').trim(); final String topic = commandLine.getOptionValue('t').trim(); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; final boolean showAll = commandLine.hasOption('a'); if (commandLine.hasOption('g') && commandLine.hasOption('d')) { final String consumerGroup = commandLine.getOptionValue('g').trim(); @@ -189,14 +191,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } if (consumerRunningInfo != null && ConsumerRunningInfo.isPushType(consumerRunningInfo)) { ConsumeMessageDirectlyResult result = - defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); + defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); System.out.printf("%s", result); } else { - System.out.printf("get consumer info failed or this %s client is not push consumer ,not support direct push \n", clientId); + System.out.printf("get consumer info failed or this %s client is not push consumer, not support direct push \n", clientId); } } else { - queryById(defaultMQAdminExt, topic, msgId, showAll); + queryById(defaultMQAdminExt, clusterName, topic, msgId, showAll); } } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java index 1d81287ac7d..f2803b0cbb3 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.tools.command.metadata; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; @@ -33,10 +32,13 @@ import java.io.File; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; public class RocksDBConfigToJsonCommand implements SubCommand { private static final String TOPICS_JSON_CONFIG = "topics"; private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; + private static final String CONSUMER_OFFSETS_JSON_CONFIG = "consumerOffsets"; @Override public String commandName() { @@ -45,7 +47,7 @@ public String commandName() { @Override public String commandDesc() { - return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; + return "Convert RocksDB kv config (topics/subscriptionGroups/consumerOffsets) to json"; } @Override @@ -56,7 +58,7 @@ public Options buildCommandlineOptions(Options options) { options.addOption(pathOption); Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + - "topics/subscriptionGroups"); + "topics/subscriptionGroups/consumerOffsets"); configTypeOption.setRequired(true); options.addOption(configTypeOption); @@ -71,19 +73,21 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t return; } - String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + String configType = commandLine.getOptionValue("configType").trim(); if (!path.endsWith("/")) { path += "/"; } path += configType; - + if (CONSUMER_OFFSETS_JSON_CONFIG.equalsIgnoreCase(configType)) { + printConsumerOffsets(path); + return; + } ConfigRocksDBStorage configRocksDBStorage = new ConfigRocksDBStorage(path, true); configRocksDBStorage.start(); RocksIterator iterator = configRocksDBStorage.iterator(); - try { final Map configMap = new HashMap<>(); - final Map configTable = new HashMap<>(); + final JSONObject configTable = new JSONObject(); iterator.seekToFirst(); while (iterator.isValid()) { final byte[] key = iterator.key(); @@ -95,14 +99,16 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t iterator.next(); } byte[] kvDataVersion = configRocksDBStorage.getKvDataVersion(); - configMap.put("dataVersion", - JSONObject.parseObject(new String(kvDataVersion, DataConverter.CHARSET_UTF8))); + if (kvDataVersion != null) { + configMap.put("dataVersion", + JSONObject.parseObject(new String(kvDataVersion, DataConverter.CHARSET_UTF8))); + } - if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { - configMap.put("topicConfigTable", JSON.parseObject(JSONObject.toJSONString(configTable))); + if (TOPICS_JSON_CONFIG.equalsIgnoreCase(configType)) { + configMap.put("topicConfigTable", configTable); } - if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { - configMap.put("subscriptionGroupTable", JSON.parseObject(JSONObject.toJSONString(configTable))); + if (SUBSCRIPTION_GROUP_JSON_CONFIG.equalsIgnoreCase(configType)) { + configMap.put("subscriptionGroupTable", configTable); } System.out.print(JSONObject.toJSONString(configMap, true) + "\n"); } catch (Exception e) { @@ -111,4 +117,42 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t configRocksDBStorage.shutdown(); } } + + private void printConsumerOffsets(String path) { + ConfigRocksDBStorage configRocksDBStorage = new ConfigRocksDBStorage(path, true); + configRocksDBStorage.start(); + RocksIterator iterator = configRocksDBStorage.iterator(); + try { + final Map configMap = new HashMap<>(); + final JSONObject configTable = new JSONObject(); + iterator.seekToFirst(); + while (iterator.isValid()) { + final byte[] key = iterator.key(); + final byte[] value = iterator.value(); + final String name = new String(key, DataConverter.CHARSET_UTF8); + final String config = new String(value, DataConverter.CHARSET_UTF8); + final RocksDBOffsetSerializeWrapper jsonObject = JSONObject.parseObject(config, RocksDBOffsetSerializeWrapper.class); + configTable.put(name, jsonObject.getOffsetTable()); + iterator.next(); + } + configMap.put("offsetTable", configTable); + System.out.print(JSONObject.toJSONString(configMap, true) + "\n"); + } catch (Exception e) { + System.out.print("Error occurred while converting RocksDB kv config to json, " + "configType=consumerOffsets, " + e.getMessage() + "\n"); + } finally { + configRocksDBStorage.shutdown(); + } + } + + static class RocksDBOffsetSerializeWrapper { + private ConcurrentMap offsetTable = new ConcurrentHashMap<>(16); + + public ConcurrentMap getOffsetTable() { + return offsetTable; + } + + public void setOffsetTable(ConcurrentMap offsetTable) { + this.offsetTable = offsetTable; + } + } } \ No newline at end of file diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java index 993fa501875..84a301bd60c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java @@ -77,6 +77,10 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -88,6 +92,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t String group = commandLine.getOptionValue("g").trim(); String topic = commandLine.getOptionValue("t").trim(); String timeStampStr = commandLine.getOptionValue("s").trim(); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; long timestamp = "now".equals(timeStampStr) ? System.currentTimeMillis() : 0; try { @@ -129,7 +134,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t if (brokerAddr != null && queueId >= 0) { System.out.printf("start reset consumer offset by specified, " + "group[%s], topic[%s], queueId[%s], broker[%s], timestamp(string)[%s], timestamp(long)[%s]%n", - group, topic, queueId, brokerAddr, timeStampStr, timestamp); + group, topic, queueId, brokerAddr, timeStampStr, timestamp); long resetOffset = null != offset ? offset : defaultMQAdminExt.searchOffset(brokerAddr, topic, queueId, timestamp, 3000); @@ -143,11 +148,11 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t Map offsetTable; try { - offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(topic, group, timestamp, force, isC); + offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(clusterName, topic, group, timestamp, force, isC); } catch (MQClientException e) { - // if consumer not online, use old command to reset reset + // if consumer not online, use old command to reset if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { - ResetOffsetByTimeOldCommand.resetOffset(defaultMQAdminExt, group, topic, timestamp, force, timeStampStr); + ResetOffsetByTimeOldCommand.resetOffset(defaultMQAdminExt, clusterName, group, topic, timestamp, force, timeStampStr); return; } throw e; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java index 7984bb8c39f..c179c5c8051 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java @@ -34,12 +34,13 @@ public class ResetOffsetByTimeOldCommand implements SubCommand { - public static void resetOffset(DefaultMQAdminExt defaultMQAdminExt, String consumerGroup, String topic, + public static void resetOffset(DefaultMQAdminExt defaultMQAdminExt, String clusterName, String consumerGroup, + String topic, long timestamp, boolean force, String timeStampStr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { List rollbackStatsList = - defaultMQAdminExt.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force); + defaultMQAdminExt.resetOffsetByTimestampOld(clusterName, consumerGroup, topic, timestamp, force); System.out.printf("reset consumer offset by specified " + "consumerGroup[%s], topic[%s], force[%s], timestamp(string)[%s], timestamp(long)[%s]%n", @@ -93,6 +94,11 @@ public Options buildCommandlineOptions(Options options) { opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]"); opt.setRequired(false); options.addOption(opt); + + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -104,6 +110,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t String consumerGroup = commandLine.getOptionValue("g").trim(); String topic = commandLine.getOptionValue("t").trim(); String timeStampStr = commandLine.getOptionValue("s").trim(); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; long timestamp = 0; try { timestamp = Long.parseLong(timeStampStr); @@ -123,7 +130,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t force = Boolean.parseBoolean(commandLine.getOptionValue("f").trim()); } defaultMQAdminExt.start(); - resetOffset(defaultMQAdminExt, consumerGroup, topic, timestamp, force, timeStampStr); + resetOffset(defaultMQAdminExt, clusterName, consumerGroup, topic, timestamp, force, timeStampStr); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java index b22491a5918..8f2ac2e1e14 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java @@ -57,6 +57,10 @@ public Options buildCommandlineOptions(Options options) { opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]"); opt.setRequired(false); options.addOption(opt); + + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -68,6 +72,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t try { String group = commandLine.getOptionValue("g").trim(); String topic = commandLine.getOptionValue("t").trim(); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; boolean force = true; if (commandLine.hasOption('f')) { force = Boolean.valueOf(commandLine.getOptionValue("f").trim()); @@ -76,7 +81,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t defaultMQAdminExt.start(); Map offsetTable; try { - offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(topic, group, timestamp, force); + offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(clusterName, topic, group, timestamp, force); } catch (MQClientException e) { if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { List rollbackStatsList = defaultMQAdminExt.resetOffsetByTimestampOld(group, topic, timestamp, force); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java new file mode 100644 index 00000000000..d18a24ee1dc --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.queue; + +import java.util.Map; +import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.SubCommand; + +public class CheckRocksdbCqWriteProgressCommand implements SubCommand { + + @Override + public String commandName() { + return "checkRocksdbCqWriteProgress"; + } + + @Override + public String commandDesc() { + return "check if rocksdb cq is same as file cq"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("c", "cluster", true, "cluster name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("n", "nameserverAddr", true, "nameserverAddr"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + defaultMQAdminExt.setNamesrvAddr(StringUtils.trim(commandLine.getOptionValue('n'))); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : ""; + String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : ""; + + try { + defaultMQAdminExt.start(); + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Map brokerAddrTable = clusterInfo.getBrokerAddrTable(); + if (clusterAddrTable.get(clusterName) == null) { + System.out.print("clusterAddrTable is empty"); + return; + } + for (Map.Entry entry : brokerAddrTable.entrySet()) { + String brokerName = entry.getKey(); + BrokerData brokerData = entry.getValue(); + String brokerAddr = brokerData.getBrokerAddrs().get(0L); + CheckRocksdbCqWriteProgressResponseBody body = defaultMQAdminExt.checkRocksdbCqWriteProgress(brokerAddr, topic); + if (StringUtils.isNotBlank(topic)) { + System.out.print(body.getDiffResult()); + } else { + System.out.print(brokerName + " | " + brokerAddr + " | \n" + body.getDiffResult()); + } + } + + } catch (Exception e) { + throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java index a1619ecedfd..47ca761d1f6 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java @@ -27,6 +27,8 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; @@ -48,6 +50,10 @@ public Options buildCommandlineOptions(Options options) { Option opt = new Option("t", "topic", true, "topic name"); opt.setRequired(true); options.addOption(opt); + + opt = new Option("c", "cluster", true, "cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -58,10 +64,26 @@ public void execute(final CommandLine commandLine, final Options options, defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + TopicStatsTable topicStatsTable = new TopicStatsTable(); defaultMQAdminExt.start(); String topic = commandLine.getOptionValue('t').trim(); - TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic); + + if (commandLine.hasOption('c')) { + String cluster = commandLine.getOptionValue('c').trim(); + TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(cluster); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + TopicStatsTable tst = defaultMQAdminExt.examineTopicStats(addr, topic); + topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable()); + } + } + } else { + topicStatsTable = defaultMQAdminExt.examineTopicStats(topic); + } List mqList = new LinkedList<>(); mqList.addAll(topicStatsTable.getOffsetTable().keySet()); diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java index fc5405e7472..b24bd22db8f 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java @@ -127,7 +127,7 @@ public void before() throws NoSuchFieldException, IllegalAccessException, Interr when(mQAdminImpl.queryMessageByUniqKey(anyString(), anyString())).thenReturn(retMsgExt); QueryResult queryResult = new QueryResult(0, Lists.newArrayList(retMsgExt)); - when(defaultMQAdminExtImpl.queryMessageByUniqKey(anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(queryResult); + when(mQAdminImpl.queryMessageByUniqKey(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(queryResult); TopicRouteData topicRouteData = new TopicRouteData(); List brokerDataList = new ArrayList<>(); @@ -194,7 +194,7 @@ public void testExecuteConsumeActively() throws SubCommandException, Interrupted Options options = ServerUtil.buildCommandlineOptions(new Options()); - String[] args = new String[] {"-t myTopicTest", "-i msgId"}; + String[] args = new String[] {"-t myTopicTest", "-i msgId", "-c DefaultCluster"}; CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); @@ -218,7 +218,7 @@ public void testExecuteConsumePassively() throws SubCommandException, Interrupte Options options = ServerUtil.buildCommandlineOptions(new Options()); - String[] args = new String[] {"-t myTopicTest", "-i 7F000001000004D20000000000000066"}; + String[] args = new String[] {"-t myTopicTest", "-i 7F000001000004D20000000000000066", "-c DefaultCluster"}; CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); @@ -230,7 +230,7 @@ public void testExecuteWithConsumerGroupAndClientId() throws SubCommandException Options options = ServerUtil.buildCommandlineOptions(new Options()); - String[] args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId"}; + String[] args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId", "-c DefaultCluster"}; CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); @@ -241,13 +241,13 @@ public void testExecute() throws SubCommandException { System.setProperty("rocketmq.namesrv.addr", "127.0.0.1:9876"); - String[] args = new String[]{"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000"}; + String[] args = new String[]{"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-c DefaultCluster"}; Options options = ServerUtil.buildCommandlineOptions(new Options()); CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); - args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId"}; + args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId", "-c DefaultCluster"}; commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null);