From 820cebee92bf82016643a972bf2889954fada65a Mon Sep 17 00:00:00 2001 From: Danyal-Faheem Date: Wed, 21 Aug 2024 18:25:33 +0500 Subject: [PATCH 1/7] feat: add do command to update the authentication plugin of MySQL users to caching_sha2_password closes #1095 --- ...heem_mysql_authentication_plugin_change.md | 1 + docs/local.rst | 20 ++++++ docs/troubleshooting.rst | 7 ++ tests/commands/test_jobs.py | 21 ++++++ tutor/commands/jobs.py | 41 +++++++++++- tutor/templates/k8s/deployments.yml | 2 +- tutor/templates/local/docker-compose.yml | 2 +- tutor/utils.py | 66 +++++++++++++++++++ 8 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20240718_171945_danyal.faheem_mysql_authentication_plugin_change.md diff --git a/changelog.d/20240718_171945_danyal.faheem_mysql_authentication_plugin_change.md b/changelog.d/20240718_171945_danyal.faheem_mysql_authentication_plugin_change.md new file mode 100644 index 0000000000..907712997b --- /dev/null +++ b/changelog.d/20240718_171945_danyal.faheem_mysql_authentication_plugin_change.md @@ -0,0 +1 @@ +- [Improvement] Add a do command to update the authentication plugin of existing MySQL users from mysql_native_password to caching_sha2_password for compatibility with MySQL v8.4.0 and above. (by @Danyal-Faheem) diff --git a/docs/local.rst b/docs/local.rst index 7f3ebcb307..3f0ebfc210 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -141,6 +141,26 @@ The default Open edX theme is rather bland, so Tutor makes it easy to switch to Out of the box, only the default "open-edx" theme is available. We also developed `Indigo, a beautiful, customizable theme `__ which is easy to install with Tutor. +.. _update_mysql_authentication_plugin: + +Updating the authentication plugin of MySQL users +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As of MySQL v8.4.0, the ``mysql_native_password`` authentication plugin has been deprecated. Users created with this authentication plugin should ideally be updated to use the latest ``caching_sha2_password`` authentication plugin. + +Tutor makes it easy do so with this handy command:: + + tutor local do update_mysql_authentication_plugin + +If you only want to update the authentication plugin of specific users, you can use the ``--users`` option. This option takes comma seperated names of users to upgrade:: + + tutor local do update_mysql_authentication_plugin --users=discovery,ecommerce + +Do note that if you are updating a specific user, there should be corresponding entries in the configuration for the mysql username and password for that user. For example, if you are trying to update the user ``myuser``, the following case sensitive entries need to be present in the configuration:: + + MYUSER_MYSQL_USERNAME + MYUSER_MYSQL_PASSWORD + Running arbitrary ``manage.py`` commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index a29935541a..dba57fac01 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -216,3 +216,10 @@ NPM Dependency Conflict When overriding ``@edx/frontend-component-header`` or `` ---------------------------------------------------------------------------------------------------------------- The detailed steps are mentioned in `tutor-mfe `__ documentation. + +"Plugin 'mysql_native_password' is not loaded" +---------------------------------------------- + +This issue can occur when Tutor is upgraded from v15 (Olive) or earlier to v18 (Redwood) because the users created in Tutor v15 utilize the mysql_native_password authentication plugin by default. This plugin has been deprecated as of MySQL v8.4.0 which is the default MySQL server used in Tutor v18. + +The handy :ref:`update_mysql_authentication_plugin ` do command in tutor can be used to fix this issue. \ No newline at end of file diff --git a/tests/commands/test_jobs.py b/tests/commands/test_jobs.py index 2ab388773b..bbd3aaadca 100644 --- a/tests/commands/test_jobs.py +++ b/tests/commands/test_jobs.py @@ -90,3 +90,24 @@ def test_set_theme(self) -> None: self.assertIn("lms-job", dc_args) self.assertIn("assign_theme('beautiful', 'domain1')", dc_args[-1]) self.assertIn("assign_theme('beautiful', 'domain2')", dc_args[-1]) + + def test_update_mysql_authentication_plugin(self) -> None: + with temporary_root() as root: + self.invoke_in_root(root, ["config", "save"]) + with patch("tutor.utils.docker_compose") as mock_docker_compose: + result = self.invoke_in_root( + root, + [ + "local", + "do", + "update-mysql-authentication-plugin", + ], + ) + dc_args, _dc_kwargs = mock_docker_compose.call_args + + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn("lms-job", dc_args) + self.assertIn("caching_sha2_password", dc_args[-1]) + self.assertIn("openedx", dc_args[-1]) + self.assertIn("root", dc_args[-1]) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 7510a83b31..4a3cc5bfaa 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -12,8 +12,9 @@ from typing_extensions import ParamSpec from tutor import config as tutor_config -from tutor import env, fmt, hooks +from tutor import env, fmt, hooks, plugins from tutor.hooks import priorities +from tutor.utils import get_mysql_change_authentication_plugin_query class DoGroup(click.Group): @@ -315,6 +316,43 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]: yield ("lms", command) +@click.command(context_settings={"ignore_unknown_options": True}) +@click.option( + "--users", + is_flag=False, + nargs=1, + help="Specific users to upgrade the authentication plugin of. Requires comma-seperated values with no space in-between.", +) +def update_mysql_authentication_plugin(users: str) -> t.Iterable[tuple[str, str]]: + """ + Update the authentication plugin of MySQL users from mysql_native_password to caching_sha2_password + Handy command used when upgrading to v8.4 of MySQL which deprecates mysql_native_password + """ + + context = click.get_current_context().obj + config = tutor_config.load(context.root) + + if not config["RUN_MYSQL"]: + fmt.echo_info( + f"You are not running MySQL (RUN_MYSQL=False). It is your " + f"responsibility to update the authentication plugin of mysql users." + ) + return + + users_to_update = users.split(",") if users else list(plugins.iter_loaded()) + + query = get_mysql_change_authentication_plugin_query( + config, users_to_update, not users + ) + + mysql_command = ( + "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --database={{ OPENEDX_MYSQL_DATABASE }} " + + shlex.join(["-e", query]) + ) + + yield ("lms", mysql_command) + + def add_job_commands(do_command_group: click.Group) -> None: """ This is meant to be called with the `local/dev/k8s do` group commands, to add the @@ -397,5 +435,6 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None: print_edx_platform_setting, settheme, sqlshell, + update_mysql_authentication_plugin, ] ) diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 778118b511..cfc62cf64d 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -397,7 +397,7 @@ spec: - "--character-set-server=utf8mb4" - "--collation-server=utf8mb4_unicode_ci" - "--binlog-expire-logs-seconds=259200" - - "--mysql-native-password=ON" + {% if DOCKER_IMAGE_MYSQL >= "docker.io/mysql:8.4.0" -%}- "--mysql-native-password=ON"{%- endif %} env: - name: MYSQL_ROOT_PASSWORD value: "{{ MYSQL_ROOT_PASSWORD }}" diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 14e141434d..7b172e53aa 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -44,7 +44,7 @@ services: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --binlog-expire-logs-seconds=259200 - --mysql-native-password=ON + {% if DOCKER_IMAGE_MYSQL >= "docker.io/mysql:8.4.0" -%}--mysql-native-password=ON{%- endif %} restart: unless-stopped user: "999:999" volumes: diff --git a/tutor/utils.py b/tutor/utils.py index cdb082f38a..984a8f8748 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -20,6 +20,7 @@ from Crypto.PublicKey.RSA import RsaKey from . import exceptions, fmt +from tutor.types import Config, ConfigValue def encrypt(text: str) -> str: @@ -366,3 +367,68 @@ def format_table(rows: List[Tuple[str, ...]], separator: str = "\t") -> str: # Append EOL at all lines but the last one formatted += "\n" return formatted + + +def get_mysql_change_authentication_plugin_query( + config: Config, users: List[str], all_users: bool +) -> str: + """ + Generates SQL queries to update the authentication plugin for MySQL users. + + This method constructs queries to change the authentication plugin to + `caching_sha2_password`. User credentials must be provided in the tutor + configuration under the keys `_MYSQL_USERNAME` and `_MYSQL_PASSWORD`. + + Args: + config (Config): Tutor configuration object + users (List[str]): List of specific MySQL users to update. + all_users (bool): Flag indicating whether to include ROOT and OPENEDX users + in addition to those specified in the `users` list. + + Returns: + str: A string containing the SQL queries to execute. + + Raises: + TutorError: If any user in the `users` list does not have corresponding + username or password entries in the configuration. + """ + + host = "%" + query = "" + + def generate_mysql_authentication_plugin_update_query( + username: ConfigValue, password: ConfigValue, host: str + ) -> str: + return f"ALTER USER '{username}'@'{host}' IDENTIFIED with caching_sha2_password BY '{password}';" + + def generate_user_queries(users: List[str]) -> str: + query = "" + for user in users: + user_uppercase = user.upper() + if not ( + f"{user_uppercase}_MYSQL_USERNAME" in config + and f"{user_uppercase}_MYSQL_PASSWORD" in config + ): + raise exceptions.TutorError( + f"Username or Password for User {user} not found in config. " + f"Please make sure that the following entries are present in the configuration:\n" + f"{user_uppercase}_MYSQL_USERNAME\n{user_uppercase}_MYSQL_PASSWORD" + ) + query += generate_mysql_authentication_plugin_update_query( + config[f"{user_uppercase}_MYSQL_USERNAME"], + config[f"{user_uppercase}_MYSQL_PASSWORD"], + host, + ) + return query + + if not all_users: + return generate_user_queries(users) + + query += generate_mysql_authentication_plugin_update_query( + config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"], host + ) + query += generate_mysql_authentication_plugin_update_query( + config["OPENEDX_MYSQL_USERNAME"], config["OPENEDX_MYSQL_PASSWORD"], host + ) + + return query + generate_user_queries(users) From 4d0042c02b080138588f0ce94bf9ccbfc87c1ac3 Mon Sep 17 00:00:00 2001 From: Danyal-Faheem Date: Fri, 30 Aug 2024 19:37:04 +0500 Subject: [PATCH 2/7] fix: change lexicographical based comparison to numerical comparison to handle edge cases --- tutor/templates/k8s/deployments.yml | 6 +++++- tutor/templates/local/docker-compose.yml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index cfc62cf64d..fe1fd415d9 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -397,7 +397,11 @@ spec: - "--character-set-server=utf8mb4" - "--collation-server=utf8mb4_unicode_ci" - "--binlog-expire-logs-seconds=259200" - {% if DOCKER_IMAGE_MYSQL >= "docker.io/mysql:8.4.0" -%}- "--mysql-native-password=ON"{%- endif %} + # We only require this option for MySQL 8.4 and above + # Breaks MySQL for previous versions as this option does not exist on versions earlier than 8.4 + {% if DOCKER_IMAGE_MYSQL.split(':')[-1].split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list -%} + - "--mysql-native-password=ON" + {%- endif %} env: - name: MYSQL_ROOT_PASSWORD value: "{{ MYSQL_ROOT_PASSWORD }}" diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 7b172e53aa..3b764fdd99 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -44,7 +44,11 @@ services: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --binlog-expire-logs-seconds=259200 - {% if DOCKER_IMAGE_MYSQL >= "docker.io/mysql:8.4.0" -%}--mysql-native-password=ON{%- endif %} + # We only require this option for MySQL 8.4 and above + # Breaks MySQL for previous versions as this option does not exist on versions earlier than 8.4 + {% if DOCKER_IMAGE_MYSQL.split(':')[-1].split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list -%} + --mysql-native-password=ON + {%- endif %} restart: unless-stopped user: "999:999" volumes: From 19802ea83d1553876952a4e873392698cc8bf044 Mon Sep 17 00:00:00 2001 From: Danyal-Faheem Date: Mon, 2 Sep 2024 11:45:02 +0500 Subject: [PATCH 3/7] fix: add condition for latest image tag as well --- tutor/templates/k8s/deployments.yml | 2 +- tutor/templates/local/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index fe1fd415d9..bd167df1f7 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -399,7 +399,7 @@ spec: - "--binlog-expire-logs-seconds=259200" # We only require this option for MySQL 8.4 and above # Breaks MySQL for previous versions as this option does not exist on versions earlier than 8.4 - {% if DOCKER_IMAGE_MYSQL.split(':')[-1].split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list -%} + {% if DOCKER_IMAGE_MYSQL.split(':')[-1] == "latest" or (DOCKER_IMAGE_MYSQL.split(':')[-1].split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list) -%} - "--mysql-native-password=ON" {%- endif %} env: diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 3b764fdd99..ff09e30995 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -46,7 +46,7 @@ services: --binlog-expire-logs-seconds=259200 # We only require this option for MySQL 8.4 and above # Breaks MySQL for previous versions as this option does not exist on versions earlier than 8.4 - {% if DOCKER_IMAGE_MYSQL.split(':')[-1].split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list -%} + {% if DOCKER_IMAGE_MYSQL.split(':')[-1] == "latest" or (DOCKER_IMAGE_MYSQL.split(':')[-1].split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list) -%} --mysql-native-password=ON {%- endif %} restart: unless-stopped From a9e987e814a6fecdb57f583292258d32456f1b68 Mon Sep 17 00:00:00 2001 From: Danyal-Faheem Date: Wed, 4 Sep 2024 13:47:45 +0500 Subject: [PATCH 4/7] refactor: separate code into another module --- tutor/commands/jobs.py | 7 ++- tutor/commands/jobs_utils.py | 74 ++++++++++++++++++++++++ tutor/templates/local/docker-compose.yml | 7 ++- tutor/utils.py | 64 -------------------- 4 files changed, 82 insertions(+), 70 deletions(-) create mode 100644 tutor/commands/jobs_utils.py diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 4a3cc5bfaa..ca51073884 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -13,8 +13,9 @@ from tutor import config as tutor_config from tutor import env, fmt, hooks, plugins +from tutor.commands.context import Context +from tutor.commands.jobs_utils import get_mysql_change_authentication_plugin_query from tutor.hooks import priorities -from tutor.utils import get_mysql_change_authentication_plugin_query class DoGroup(click.Group): @@ -323,13 +324,13 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]: nargs=1, help="Specific users to upgrade the authentication plugin of. Requires comma-seperated values with no space in-between.", ) -def update_mysql_authentication_plugin(users: str) -> t.Iterable[tuple[str, str]]: +@click.pass_obj +def update_mysql_authentication_plugin(context: Context, users: str) -> t.Iterable[tuple[str, str]]: """ Update the authentication plugin of MySQL users from mysql_native_password to caching_sha2_password Handy command used when upgrading to v8.4 of MySQL which deprecates mysql_native_password """ - context = click.get_current_context().obj config = tutor_config.load(context.root) if not config["RUN_MYSQL"]: diff --git a/tutor/commands/jobs_utils.py b/tutor/commands/jobs_utils.py new file mode 100644 index 0000000000..dfc539ca13 --- /dev/null +++ b/tutor/commands/jobs_utils.py @@ -0,0 +1,74 @@ +""" +This module provides utility methods for tutor `do` commands + +Methods: +- `get_mysql_change_authentication_plugin_query`: Generates MySQL queries to update the authentication plugin for MySQL users. +""" +from typing import List + +from tutor import exceptions +from tutor.types import Config, ConfigValue + +def get_mysql_change_authentication_plugin_query( + config: Config, users: List[str], all_users: bool +) -> str: + """ + Generates MySQL queries to update the authentication plugin for MySQL users. + + This method constructs queries to change the authentication plugin to + `caching_sha2_password`. User credentials must be provided in the tutor + configuration under the keys `_MYSQL_USERNAME` and `_MYSQL_PASSWORD`. + + Args: + config (Config): Tutor configuration object + users (List[str]): List of specific MySQL users to update. + all_users (bool): Flag indicating whether to include ROOT and OPENEDX users + in addition to those specified in the `users` list. + + Returns: + str: A string containing the SQL queries to execute. + + Raises: + TutorError: If any user in the `users` list does not have corresponding + username or password entries in the configuration. + """ + + host = "%" + query = "" + + def generate_mysql_authentication_plugin_update_query( + username: ConfigValue, password: ConfigValue, host: str + ) -> str: + return f"ALTER USER '{username}'@'{host}' IDENTIFIED with caching_sha2_password BY '{password}';" + + def generate_user_queries(users: List[str]) -> str: + query = "" + for user in users: + user_uppercase = user.upper() + if not ( + f"{user_uppercase}_MYSQL_USERNAME" in config + and f"{user_uppercase}_MYSQL_PASSWORD" in config + ): + raise exceptions.TutorError( + f"Username or Password for User {user} not found in config. " + f"Please make sure that the following entries are present in the configuration:\n" + f"{user_uppercase}_MYSQL_USERNAME\n{user_uppercase}_MYSQL_PASSWORD" + ) + query += generate_mysql_authentication_plugin_update_query( + config[f"{user_uppercase}_MYSQL_USERNAME"], + config[f"{user_uppercase}_MYSQL_PASSWORD"], + host, + ) + return query + + if not all_users: + return generate_user_queries(users) + + query += generate_mysql_authentication_plugin_update_query( + config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"], host + ) + query += generate_mysql_authentication_plugin_update_query( + config["OPENEDX_MYSQL_USERNAME"], config["OPENEDX_MYSQL_PASSWORD"], host + ) + + return query + generate_user_queries(users) diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index ff09e30995..9acc747916 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -44,9 +44,10 @@ services: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --binlog-expire-logs-seconds=259200 - # We only require this option for MySQL 8.4 and above - # Breaks MySQL for previous versions as this option does not exist on versions earlier than 8.4 - {% if DOCKER_IMAGE_MYSQL.split(':')[-1] == "latest" or (DOCKER_IMAGE_MYSQL.split(':')[-1].split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list) -%} + # We only require this option for MySQL 8.4.0 and above + # Breaks MySQL for previous versions as this option does not exist on versions earlier than 8.4.0 + {% set mysql_version = DOCKER_IMAGE_MYSQL.split(':')[-1] -%} + {% if mysql_version == "latest" or (mysql_version.split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list) -%} --mysql-native-password=ON {%- endif %} restart: unless-stopped diff --git a/tutor/utils.py b/tutor/utils.py index 984a8f8748..44d8ff6abe 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -20,7 +20,6 @@ from Crypto.PublicKey.RSA import RsaKey from . import exceptions, fmt -from tutor.types import Config, ConfigValue def encrypt(text: str) -> str: @@ -369,66 +368,3 @@ def format_table(rows: List[Tuple[str, ...]], separator: str = "\t") -> str: return formatted -def get_mysql_change_authentication_plugin_query( - config: Config, users: List[str], all_users: bool -) -> str: - """ - Generates SQL queries to update the authentication plugin for MySQL users. - - This method constructs queries to change the authentication plugin to - `caching_sha2_password`. User credentials must be provided in the tutor - configuration under the keys `_MYSQL_USERNAME` and `_MYSQL_PASSWORD`. - - Args: - config (Config): Tutor configuration object - users (List[str]): List of specific MySQL users to update. - all_users (bool): Flag indicating whether to include ROOT and OPENEDX users - in addition to those specified in the `users` list. - - Returns: - str: A string containing the SQL queries to execute. - - Raises: - TutorError: If any user in the `users` list does not have corresponding - username or password entries in the configuration. - """ - - host = "%" - query = "" - - def generate_mysql_authentication_plugin_update_query( - username: ConfigValue, password: ConfigValue, host: str - ) -> str: - return f"ALTER USER '{username}'@'{host}' IDENTIFIED with caching_sha2_password BY '{password}';" - - def generate_user_queries(users: List[str]) -> str: - query = "" - for user in users: - user_uppercase = user.upper() - if not ( - f"{user_uppercase}_MYSQL_USERNAME" in config - and f"{user_uppercase}_MYSQL_PASSWORD" in config - ): - raise exceptions.TutorError( - f"Username or Password for User {user} not found in config. " - f"Please make sure that the following entries are present in the configuration:\n" - f"{user_uppercase}_MYSQL_USERNAME\n{user_uppercase}_MYSQL_PASSWORD" - ) - query += generate_mysql_authentication_plugin_update_query( - config[f"{user_uppercase}_MYSQL_USERNAME"], - config[f"{user_uppercase}_MYSQL_PASSWORD"], - host, - ) - return query - - if not all_users: - return generate_user_queries(users) - - query += generate_mysql_authentication_plugin_update_query( - config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"], host - ) - query += generate_mysql_authentication_plugin_update_query( - config["OPENEDX_MYSQL_USERNAME"], config["OPENEDX_MYSQL_PASSWORD"], host - ) - - return query + generate_user_queries(users) From d266f4bfb658ab6f160c12678c3fc2a10e631620 Mon Sep 17 00:00:00 2001 From: Danyal-Faheem Date: Wed, 4 Sep 2024 13:48:56 +0500 Subject: [PATCH 5/7] fix: format files --- tutor/commands/jobs.py | 4 +++- tutor/commands/jobs_utils.py | 2 ++ tutor/utils.py | 2 -- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index ca51073884..90451536f0 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -325,7 +325,9 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]: help="Specific users to upgrade the authentication plugin of. Requires comma-seperated values with no space in-between.", ) @click.pass_obj -def update_mysql_authentication_plugin(context: Context, users: str) -> t.Iterable[tuple[str, str]]: +def update_mysql_authentication_plugin( + context: Context, users: str +) -> t.Iterable[tuple[str, str]]: """ Update the authentication plugin of MySQL users from mysql_native_password to caching_sha2_password Handy command used when upgrading to v8.4 of MySQL which deprecates mysql_native_password diff --git a/tutor/commands/jobs_utils.py b/tutor/commands/jobs_utils.py index dfc539ca13..157edac664 100644 --- a/tutor/commands/jobs_utils.py +++ b/tutor/commands/jobs_utils.py @@ -4,11 +4,13 @@ Methods: - `get_mysql_change_authentication_plugin_query`: Generates MySQL queries to update the authentication plugin for MySQL users. """ + from typing import List from tutor import exceptions from tutor.types import Config, ConfigValue + def get_mysql_change_authentication_plugin_query( config: Config, users: List[str], all_users: bool ) -> str: diff --git a/tutor/utils.py b/tutor/utils.py index 44d8ff6abe..cdb082f38a 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -366,5 +366,3 @@ def format_table(rows: List[Tuple[str, ...]], separator: str = "\t") -> str: # Append EOL at all lines but the last one formatted += "\n" return formatted - - From 24e88725bcc95a0bc79c0d2169eb79cd794c7415 Mon Sep 17 00:00:00 2001 From: Danyal-Faheem Date: Mon, 9 Sep 2024 13:40:47 +0500 Subject: [PATCH 6/7] fix: make requested changes --- docs/local.rst | 4 ++-- tutor/commands/jobs.py | 6 +++++- tutor/commands/jobs_utils.py | 13 ++++++++----- tutor/fmt.py | 8 ++++++++ tutor/templates/k8s/deployments.yml | 7 ++++--- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/docs/local.rst b/docs/local.rst index 3f0ebfc210..f342171e6e 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -152,11 +152,11 @@ Tutor makes it easy do so with this handy command:: tutor local do update_mysql_authentication_plugin -If you only want to update the authentication plugin of specific users, you can use the ``--users`` option. This option takes comma seperated names of users to upgrade:: +The above command will update all the database users created by Tutor. If you only want to update the authentication plugin of specific users, you can use the ``--users`` option. This option takes comma seperated names of users to upgrade:: tutor local do update_mysql_authentication_plugin --users=discovery,ecommerce -Do note that if you are updating a specific user, there should be corresponding entries in the configuration for the mysql username and password for that user. For example, if you are trying to update the user ``myuser``, the following case sensitive entries need to be present in the configuration:: +For this command, Tutor expects specific entries in the configuration for the mysql username and password of a database user. For example, if you are trying to update the user ``myuser``, the following case sensitive entries need to be present in the configuration:: MYUSER_MYSQL_USERNAME MYUSER_MYSQL_PASSWORD diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 90451536f0..66d08d9441 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -330,7 +330,7 @@ def update_mysql_authentication_plugin( ) -> t.Iterable[tuple[str, str]]: """ Update the authentication plugin of MySQL users from mysql_native_password to caching_sha2_password - Handy command used when upgrading to v8.4 of MySQL which deprecates mysql_native_password + Handy command utilized when upgrading to v8.4 of MySQL which deprecates mysql_native_password """ config = tutor_config.load(context.root) @@ -348,6 +348,10 @@ def update_mysql_authentication_plugin( config, users_to_update, not users ) + # In case there is no user to update the authentication plugin of + if not query: + return + mysql_command = ( "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --database={{ OPENEDX_MYSQL_DATABASE }} " + shlex.join(["-e", query]) diff --git a/tutor/commands/jobs_utils.py b/tutor/commands/jobs_utils.py index 157edac664..8a4070ee4f 100644 --- a/tutor/commands/jobs_utils.py +++ b/tutor/commands/jobs_utils.py @@ -7,7 +7,7 @@ from typing import List -from tutor import exceptions +from tutor import exceptions, fmt from tutor.types import Config, ConfigValue @@ -41,6 +41,9 @@ def get_mysql_change_authentication_plugin_query( def generate_mysql_authentication_plugin_update_query( username: ConfigValue, password: ConfigValue, host: str ) -> str: + fmt.echo_info( + f"Authentication plugin of user {username} will be updated to caching_sha2_password" + ) return f"ALTER USER '{username}'@'{host}' IDENTIFIED with caching_sha2_password BY '{password}';" def generate_user_queries(users: List[str]) -> str: @@ -51,11 +54,11 @@ def generate_user_queries(users: List[str]) -> str: f"{user_uppercase}_MYSQL_USERNAME" in config and f"{user_uppercase}_MYSQL_PASSWORD" in config ): - raise exceptions.TutorError( - f"Username or Password for User {user} not found in config. " - f"Please make sure that the following entries are present in the configuration:\n" - f"{user_uppercase}_MYSQL_USERNAME\n{user_uppercase}_MYSQL_PASSWORD" + fmt.echo_warning( + f"Username or Password for User {user} not found in config. Skipping update process for User {user}." ) + continue + query += generate_mysql_authentication_plugin_update_query( config[f"{user_uppercase}_MYSQL_USERNAME"], config[f"{user_uppercase}_MYSQL_PASSWORD"], diff --git a/tutor/fmt.py b/tutor/fmt.py index 8af0592071..9cffdadd97 100644 --- a/tutor/fmt.py +++ b/tutor/fmt.py @@ -46,6 +46,14 @@ def alert(text: str) -> str: return click.style("⚠️ " + text, fg="yellow", bold=True) +def warning(text: str) -> str: + return click.style(text, fg="bright_yellow") + + +def echo_warning(text: str) -> None: + echo(warning(text)) + + def echo(text: str, err: bool = False) -> None: if os.environ.get("_TUTOR_COMPLETE"): if os.environ.get("COMP_WORDS") or os.environ.get("COMP_CWORD"): diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index bd167df1f7..166f9b9aa0 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -397,9 +397,10 @@ spec: - "--character-set-server=utf8mb4" - "--collation-server=utf8mb4_unicode_ci" - "--binlog-expire-logs-seconds=259200" - # We only require this option for MySQL 8.4 and above - # Breaks MySQL for previous versions as this option does not exist on versions earlier than 8.4 - {% if DOCKER_IMAGE_MYSQL.split(':')[-1] == "latest" or (DOCKER_IMAGE_MYSQL.split(':')[-1].split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list) -%} + # We only require this option for MySQL 8.4.0 and above + # Breaks MySQL for previous versions as this option does not exist on versions earlier than 8.4.0 + {% set mysql_version = DOCKER_IMAGE_MYSQL.split(':')[-1] -%} + {% if mysql_version == "latest" or (mysql_version.split('.') | map('int') | list >= '8.4.0'.split('.') | map('int') | list) -%} - "--mysql-native-password=ON" {%- endif %} env: From d56029f9cc28bd42937fe778313a73996dae0e52 Mon Sep 17 00:00:00 2001 From: Danyal-Faheem Date: Mon, 16 Sep 2024 12:27:38 +0500 Subject: [PATCH 7/7] docs: fix do command name --- docs/local.rst | 4 ++-- docs/troubleshooting.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/local.rst b/docs/local.rst index f342171e6e..3d1c7a14b9 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -150,11 +150,11 @@ As of MySQL v8.4.0, the ``mysql_native_password`` authentication plugin has been Tutor makes it easy do so with this handy command:: - tutor local do update_mysql_authentication_plugin + tutor local do update-mysql-authentication-plugin The above command will update all the database users created by Tutor. If you only want to update the authentication plugin of specific users, you can use the ``--users`` option. This option takes comma seperated names of users to upgrade:: - tutor local do update_mysql_authentication_plugin --users=discovery,ecommerce + tutor local do update-mysql-authentication-plugin --users=discovery,ecommerce For this command, Tutor expects specific entries in the configuration for the mysql username and password of a database user. For example, if you are trying to update the user ``myuser``, the following case sensitive entries need to be present in the configuration:: diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index dba57fac01..02355f74bf 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -222,4 +222,4 @@ The detailed steps are mentioned in `tutor-mfe ` do command in tutor can be used to fix this issue. \ No newline at end of file +The handy :ref:`update-mysql-authentication-plugin ` do command in tutor can be used to fix this issue. \ No newline at end of file