From 13e271110ebee87d52af57840a6b8be61f18f3a2 Mon Sep 17 00:00:00 2001 From: Xie Yi <105206850+xieyi888@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:58:23 +0800 Subject: [PATCH 1/7] [AMORO-3033]IndexOutOfBoundsException when start up AMS during selectProcessInputFiles because of no optimizing process_id (#3034) fix IndexOutOfBoundsException when start ams --- .../apache/amoro/server/persistence/TaskFilesPersistence.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java index 9e8f040aa6..5bbba3abdd 100644 --- a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java +++ b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java @@ -46,7 +46,7 @@ public static Map loadTaskInputs(long processId) { List bytes = persistence.getAs( OptimizingMapper.class, mapper -> mapper.selectProcessInputFiles(processId)); - if (bytes == null) { + if (bytes == null || bytes.isEmpty()) { return Collections.emptyMap(); } else { return SerializationUtil.simpleDeserialize(CompressUtil.unGzip(bytes.get(0))); From 42fe1993821aaa3a4f80f0e4fa3af932b290776d Mon Sep 17 00:00:00 2001 From: ZhouJinsong Date: Mon, 15 Jul 2024 20:16:31 +0800 Subject: [PATCH 2/7] [Hotfix] Fix build guide in README.md (#3037) Fix build guide --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0748a8832f..b06093c2ba 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Amoro is built using Maven with JDK 8 and JDK 17(only for `amoro-mixed-format/am * Build all modules without `amoro-mixed-format-trino`: `mvn clean package` * Build and skip tests: `mvn clean package -DskipTests` * Build and skip dashboard: `mvn clean package -Pskip-dashboard-build` -* Build and support disk storage, RocksDB will be introduced to avoid memory overflow: `mvn clean package -DskipTests -Psupport-disk-storage` +* Build and disable disk storage, RocksDB will NOT be introduced to avoid memory overflow: `mvn clean package -DskipTests -Pno-extented-disk-storage` * Build with hadoop 2.x(the default is 3.x) dependencies: `mvn clean package -DskipTests -Phadoop2` * Specify Flink version for Flink optimizer(the default is 1.18.1): `mvn clean package -DskipTests -Dflink-optimizer.flink-version=1.15.4` * If the version of Flink is below 1.15.0, you also need to add the `-Pflink-optimizer-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6` From a12f2984f4c0fd2017e98a7c89ba7457edb65d85 Mon Sep 17 00:00:00 2001 From: Zhengke Zhou Date: Tue, 16 Jul 2024 12:03:49 +0800 Subject: [PATCH 3/7] [AMORO-2787]: Remove `memorySize` option (#3040) remove `memorySize` option Co-authored-by: ZhouJinsong --- .../org/apache/amoro/optimizer/common/OptimizerConfig.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java index 152eb2b162..d800216ca0 100644 --- a/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java +++ b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java @@ -43,12 +43,6 @@ public class OptimizerConfig implements Serializable { required = true) private int executionParallel; - /** @deprecated This parameter is deprecated and will be removed in version 0.7.0. */ - @Deprecated - @Option( - name = "-m", - aliases = "--" + OptimizerProperties.OPTIMIZER_MEMORY_SIZE, - usage = "Optimizer memory size(MB)") private int memorySize; @Option( From 58d1f6cd37b16534b50601572c0187787bab21e7 Mon Sep 17 00:00:00 2001 From: feng xiaohang <43104233+engraving-knife@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:05:36 +0800 Subject: [PATCH 4/7] Optimize database configuration for mysql5.6. (#2961) * Optimize database configuration for mysql5.6. * Update amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql Co-authored-by: ZhouJinsong --------- Co-authored-by: xiaosefeng Co-authored-by: ZhouJinsong Co-authored-by: ConradJam --- .../main/resources/mysql/ams-mysql-init.sql | 6 +-- .../mysql/upgrade-0.4.0-to-0.4.1.sql | 2 +- .../mysql/upgrade-0.4.1-to-0.5.0.sql | 4 +- .../dist/src/main/amoro-bin/conf/config.yaml | 42 +++++++++---------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql index ce2c88ad34..82fe7072d3 100644 --- a/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql +++ b/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql @@ -82,7 +82,7 @@ CREATE TABLE `table_identifier` `format` VARCHAR(32) NOT NULL COMMENT 'Table Format', PRIMARY KEY (`table_id`), UNIQUE KEY `table_name_index` (`catalog_name`,`db_name`,`table_name`) -); +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Table identifier for AMS' ROW_FORMAT=DYNAMIC; CREATE TABLE `table_metadata` ( @@ -131,7 +131,7 @@ CREATE TABLE `table_runtime` `pending_input` mediumtext, PRIMARY KEY (`table_id`), UNIQUE KEY `table_index` (`catalog_name`,`db_name`,`table_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table' ROW_FORMAT=DYNAMIC; CREATE TABLE `table_optimizing_process` ( @@ -221,4 +221,4 @@ CREATE TABLE `table_blocker` ( `properties` mediumtext COMMENT 'Blocker properties', PRIMARY KEY (`blocker_id`), KEY `table_index` (`catalog_name`,`db_name`,`table_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers' ROW_FORMAT=DYNAMIC; diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql index 609499e114..c4d705d860 100644 --- a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql +++ b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql @@ -24,7 +24,7 @@ CREATE TABLE `table_blocker` ( `properties` mediumtext COMMENT 'Blocker properties', PRIMARY KEY (`blocker_id`), KEY `table_index` (`catalog_name`,`db_name`,`table_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers' ROW_FORMAT=DYNAMIC; ALTER TABLE `optimize_group` ADD COLUMN `scheduling_policy` varchar(20) COMMENT 'Optimize group scheduling policy' after `name`; ALTER TABLE `optimize_task` CHANGE `max_change_transaction_id` `to_sequence` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'to sequence'; diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql index 89292b95bb..f293d19587 100644 --- a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql +++ b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql @@ -59,7 +59,7 @@ CREATE TABLE `table_identifier` `table_name` varchar(128) NOT NULL COMMENT 'Table name', PRIMARY KEY (`table_id`), UNIQUE KEY `table_name_index` (`catalog_name`,`db_name`,`table_name`) -); +) ROW_FORMAT=DYNAMIC; INSERT INTO `table_identifier` (`catalog_name`, `db_name`, `table_name`) SELECT `catalog_name`, `db_name`, `table_name` FROM `table_metadata`; -- table_metadata @@ -126,7 +126,7 @@ CREATE TABLE `table_runtime` `optimizing_config` mediumtext, `pending_input` mediumtext, PRIMARY KEY (`table_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table' ROW_FORMAT=DYNAMIC; diff --git a/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml b/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml index 9402ab2fb1..cdea700842 100644 --- a/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml +++ b/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml @@ -92,27 +92,27 @@ ams: connection-pool-max-idle: 16 connection-pool-max-wait-millis: 1000 -# MySQL database configuration. -# database: -# type: mysql -# jdbc-driver-class: com.mysql.cj.jdbc.Driver -# url: jdbc:mysql://127.0.0.1:3306/db?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false -# username: root -# password: root -# connection-pool-max-total: 20 -# connection-pool-max-idle: 16 -# connection-pool-max-wait-millis: 1000 - - # Postgres database configuration. - # database: - # type: postgres - # jdbc-driver-class: org.postgresql.Driver - # url: jdbc:postgresql://127.0.0.1:5432/db - # username: user - # password: passwd - # connection-pool-max-total: 20 - # connection-pool-max-idle: 16 - # connection-pool-max-wait-millis: 1000 +# MySQL database configuration. +# database: +# type: mysql +# jdbc-driver-class: com.mysql.cj.jdbc.Driver +# url: jdbc:mysql://127.0.0.1:3306/db?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false +# username: root +# password: root +# connection-pool-max-total: 20 +# connection-pool-max-idle: 16 +# connection-pool-max-wait-millis: 1000 + +# Postgres database configuration. +# database: +# type: postgres +# jdbc-driver-class: org.postgresql.Driver +# url: jdbc:postgresql://127.0.0.1:5432/db +# username: user +# password: passwd +# connection-pool-max-total: 20 +# connection-pool-max-idle: 16 +# connection-pool-max-wait-millis: 1000 terminal: backend: local From 4f6a1d348917323a20aff1747a7215fe8f434c90 Mon Sep 17 00:00:00 2001 From: Qishang Zhong Date: Tue, 16 Jul 2024 17:55:18 +0800 Subject: [PATCH 5/7] [AMORO-3041] Config failOnNoGitDirectory and failOnUnableToExtractRepoInfo to false for git-commit-id-plugin (#3042) --- amoro-ams/amoro-ams-server/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/amoro-ams/amoro-ams-server/pom.xml b/amoro-ams/amoro-ams-server/pom.xml index 4e58dd9afc..c5f16766fe 100644 --- a/amoro-ams/amoro-ams-server/pom.xml +++ b/amoro-ams/amoro-ams-server/pom.xml @@ -387,6 +387,8 @@ + false + false ${project.basedir}/.git true From 7856711566ec9eb1d3ec64820fe2986117dc1afc Mon Sep 17 00:00:00 2001 From: ZhouJinsong Date: Tue, 16 Jul 2024 19:37:03 +0800 Subject: [PATCH 6/7] [AMORO-3043] Change source release file suffix to `tar.gz` (#3044) Change source release file suffix to tar.gz --- tools/releasing/create_source_release.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh index fd4a2cc3e8..ffb86a0fe5 100644 --- a/tools/releasing/create_source_release.sh +++ b/tools/releasing/create_source_release.sh @@ -68,10 +68,10 @@ rsync -a \ --exclude "*/dependency-reduced-pom.xml" \ . amoro-$RELEASE_VERSION -tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tgz amoro-$RELEASE_VERSION -gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tgz +tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tar.gz amoro-$RELEASE_VERSION +gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tar.gz cd ${RELEASE_DIR} -$SHASUM apache-amoro-$RELEASE_VERSION-src.tgz > apache-amoro-$RELEASE_VERSION-src.tgz.sha512 +$SHASUM apache-amoro-$RELEASE_VERSION-src.tar.gz > apache-amoro-$RELEASE_VERSION-src.tar.gz.sha512 cd ${CURR_DIR} rm -rf ${CLONE_DIR} \ No newline at end of file From bac3fb6b3613a7ea9c1076d05ca6f9237f600072 Mon Sep 17 00:00:00 2001 From: ZhouJinsong Date: Tue, 16 Jul 2024 19:58:19 +0800 Subject: [PATCH 7/7] [Hotfix] Fix liveness and readiness uri for amoro deployment (#3045) Fix liveness and readiness path for amoro deployment --- charts/amoro/templates/amoro-deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/amoro/templates/amoro-deployment.yaml b/charts/amoro/templates/amoro-deployment.yaml index da9397b719..bce9dc1bb0 100644 --- a/charts/amoro/templates/amoro-deployment.yaml +++ b/charts/amoro/templates/amoro-deployment.yaml @@ -83,7 +83,7 @@ spec: {{/* TODO If Support Replica can be use more than 1 */}} {{- if .Values.livenessProbe.enabled }} livenessProbe: httpGet: - path: /health/status + path: /ams/v1/health/status port: rest initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} @@ -94,7 +94,7 @@ spec: {{/* TODO If Support Replica can be use more than 1 */}} {{- if .Values.readinessProbe.enabled }} readinessProbe: httpGet: - path: /versionInfo + path: ams/v1/versionInfo port: rest initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }}