Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

scylla-node: prepare for the post-jmx world #565

Merged
merged 3 commits into from
Apr 2, 2024
Merged

scylla-node: prepare for the post-jmx world #565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7cad650
ccmlib/node.py: extract logic of running nodetool into separate method
denesb Mar 21, 2024
a6f1187
ccmlib/node.py: _do_run_nodetool(): filter out debug-mode warnings fr…
denesb Mar 26, 2024
1603594
ccmlib/scylla_node.py: prepare for the post-jmx world
denesb Mar 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 36 additions & 17 deletions ccmlib/node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -808,6 +808,41 @@ def wait_for_compactions(self,
time.sleep(1)
raise TimeoutError(f"Waiting for compactions timed out after {idle_timeout} seconds with pending tasks remaining: {output}.")

def _do_run_nodetool(self, nodetool, capture_output=True, wait=True, timeout=None, verbose=True):
if capture_output and not wait:
raise common.ArgumentError("Cannot set capture_output while wait is False.")
env = self.get_env()
if verbose:
self.debug(f"nodetool cmd={nodetool} wait={wait} timeout={timeout}")
if capture_output:
p = subprocess.Popen(nodetool, universal_newlines=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate(timeout=timeout)
else:
p = subprocess.Popen(nodetool, env=env, universal_newlines=True)
stdout, stderr = None, None

if wait:
exit_status = p.wait(timeout=timeout)
if exit_status != 0:
raise NodetoolError(" ".join(nodetool), exit_status, stdout, stderr)

ignored_patterns = (re.compile("WARNING: debug mode. Not for benchmarking or production"),
re.compile("==[0-9]+==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!"))

def keep(line):
self.debug(f"checking {line}")
for ignored_pattern in ignored_patterns:
if ignored_pattern.fullmatch(line):
return False
return True

def filter_debug_errors(stderr):
if stderr is None:
return stderr
return '\n'.join([line for line in stderr.split('\n') if keep(line)])

return stdout, filter_debug_errors(stderr)

def nodetool(self, cmd, capture_output=True, wait=True, timeout=None, verbose=True):
"""
Setting wait=False makes it impossible to detect errors,
Expand All @@ -816,9 +851,6 @@ def nodetool(self, cmd, capture_output=True, wait=True, timeout=None, verbose=Tr
When wait=True, timeout may be set to a number, in seconds,
to limit how long the function will wait for nodetool to complete.
"""
if capture_output and not wait:
raise common.ArgumentError("Cannot set capture_output while wait is False.")
env = self.get_env()
if self.is_scylla() and not self.is_docker():
host = self.address()
else:
Expand All @@ -830,21 +862,8 @@ def nodetool(self, cmd, capture_output=True, wait=True, timeout=None, verbose=Tr
# see https://www.oracle.com/java/technologies/javase/8u331-relnotes.html#JDK-8278972
nodetool.extend(['-h', host, '-p', str(self.jmx_port), '-Dcom.sun.jndi.rmiURLParsing=legacy'])
nodetool.extend(cmd.split())
if verbose:
self.debug(f"nodetool cmd={cmd} wait={wait} timeout={timeout}")
if capture_output:
p = subprocess.Popen(nodetool, universal_newlines=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate(timeout=timeout)
else:
p = subprocess.Popen(nodetool, env=env, universal_newlines=True)
stdout, stderr = None, None

if wait:
exit_status = p.wait(timeout=timeout)
if exit_status != 0:
raise NodetoolError(" ".join(nodetool), exit_status, stdout, stderr)

return stdout, stderr
return self._do_run_nodetool(nodetool, capture_output, wait, timeout, verbose)

def dsetool(self, cmd):
raise common.ArgumentError('Cassandra nodes do not support dsetool')
Expand Down
105 changes: 66 additions & 39 deletions ccmlib/scylla_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ def __init__(self, name, cluster, auto_bootstrap, thrift_interface,
self._node_install_dir = None
self._node_scylla_version = None
self._relative_repos_root = None
self._has_jmx = None
super().__init__(name, cluster, auto_bootstrap,
thrift_interface, storage_interface,
jmx_port, remote_debug_port,
Expand Down Expand Up @@ -83,6 +84,8 @@ def node_install_dir(self):
@node_install_dir.setter
def node_install_dir(self, install_dir):
self._node_install_dir = install_dir
# Force re-check based on the new install dir
self._has_jmx = None

@property
def node_scylla_version(self):
Expand All @@ -98,6 +101,17 @@ def node_scylla_version(self, install_dir):
def scylla_build_id(self):
return self._run_scylla_executable_with_option(option="--build-id")

@property
def has_jmx(self):
if self._has_jmx is None:
install_dir = self.node_install_dir
if self.is_scylla_reloc():
self._has_jmx = os.path.isdir(os.path.join(install_dir, "jmx"))
else:
self._has_jmx = os.path.isdir(os.path.join(install_dir, "tools", "jmx"))

return self._has_jmx

def scylla_mode(self):
return self.cluster.get_scylla_mode()

Expand Down Expand Up @@ -688,14 +702,15 @@ def process_opts(opts):
wait_normal_token_owner=wait_normal_token_owner,
wait_for_binary_proto=wait_for_binary_proto,
ext_env=ext_env)
self._start_jmx(data)
if self.has_jmx:
self._start_jmx(data)

ip_addr, _ = self.network_interfaces['storage']
jmx_port = int(self.jmx_port)
if not self._wait_java_up(ip_addr, jmx_port):
e_msg = "Error starting node {}: unable to connect to scylla-jmx port {}:{}".format(
self.name, ip_addr, jmx_port)
raise NodeError(e_msg, scylla_process)
ip_addr, _ = self.network_interfaces['storage']
jmx_port = int(self.jmx_port)
if not self._wait_java_up(ip_addr, jmx_port):
e_msg = "Error starting node {}: unable to connect to scylla-jmx port {}:{}".format(
self.name, ip_addr, jmx_port)
raise NodeError(e_msg, scylla_process)

self._update_pid(scylla_process)
wait_for(func=lambda: self.is_running(), timeout=10, step=0.01)
Expand Down Expand Up @@ -761,9 +776,19 @@ def _update_jmx_pid(self, wait=True):
self.jmx_pid = None

def nodetool(self, cmd, capture_output=True, wait=True, timeout=None, verbose=True):
"""
Kill scylla-jmx in case of timeout, to supply enough debugging information
"""
if not self.has_jmx:
if self.is_docker():
host = 'localhost'
else:
host = self.address()
nodetool = [os.path.join(self.get_bin_dir(), "scylla"), "nodetool"]
nodetool.extend(['-h', host, '-p', '10000'])
nodetool.extend(cmd.split())
return self._do_run_nodetool(nodetool, capture_output, wait, timeout, verbose)

# Fall-back to the java nodetool for pre 5.5.0~dev versions, which don't yet have the native nodetool
# Kill scylla-jmx in case of timeout, to supply enough debugging information

# pass the api_port to nodetool. if it is the nodetool-wrapper. it should
# interpret the command line and use it for the -p option
cmd = f"-Dcom.scylladb.apiPort=10000 {cmd}"
Expand Down Expand Up @@ -809,7 +834,8 @@ def do_stop(self, gently=True):
"""

did_stop = False
self._update_jmx_pid(wait=False)
if self.has_jmx:
self._update_jmx_pid(wait=False)
if self.scylla_manager and self.scylla_manager.is_agent_available:
self._update_scylla_agent_pid()
for proc in [self._process_jmx, self._process_scylla, self._process_agent]:
Expand Down Expand Up @@ -1061,34 +1087,35 @@ def run_patchelf(patchelf_cmd):
if returncode != 0:
raise RuntimeError(f'{patchelf_cmd} exited with status {returncode}.\nstdout:{stdout}\nstderr:\n{stderr}')

if 'scylla-repository' in self.node_install_dir:
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scylla-jmx-1.0.jar'),
os.path.join(self.get_bin_dir(), 'scylla-jmx-1.0.jar'), replace=replace)
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scylla-jmx'),
os.path.join(self.get_bin_dir(), 'scylla-jmx'), replace=replace)
select_java = Path(self.get_jmx_dir()) / 'select-java'
else:
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'target', 'scylla-jmx-1.0.jar'),
os.path.join(self.get_bin_dir(), 'scylla-jmx-1.0.jar'), replace=replace)
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scripts', 'scylla-jmx'),
os.path.join(self.get_bin_dir(), 'scylla-jmx'), replace=replace)
select_java = Path(self.get_jmx_dir()) / 'scripts' / 'select-java'

os.makedirs(os.path.join(self.get_bin_dir(), 'symlinks'), exist_ok=exist_ok)
scylla_jmx_file = os.path.join(self.get_bin_dir(), 'symlinks', 'scylla-jmx')
if os.path.exists(scylla_jmx_file) and replace:
os.remove(scylla_jmx_file)
if java_home := os.environ.get('JAVA_HOME'):
# user selecting specific Java
java_exe = Path(java_home) / 'bin' / 'java'
os.symlink(java_exe, scylla_jmx_file)
elif select_java.exists():
# JMX lookup logic
os.symlink(select_java, scylla_jmx_file)
else:
# older scylla versions, just use default java
java_exe = Path('/usr') / 'bin' / 'java'
os.symlink(java_exe, scylla_jmx_file)
if self.has_jmx:
if 'scylla-repository' in self.node_install_dir:
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scylla-jmx-1.0.jar'),
os.path.join(self.get_bin_dir(), 'scylla-jmx-1.0.jar'), replace=replace)
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scylla-jmx'),
os.path.join(self.get_bin_dir(), 'scylla-jmx'), replace=replace)
select_java = Path(self.get_jmx_dir()) / 'select-java'
else:
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'target', 'scylla-jmx-1.0.jar'),
os.path.join(self.get_bin_dir(), 'scylla-jmx-1.0.jar'), replace=replace)
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scripts', 'scylla-jmx'),
os.path.join(self.get_bin_dir(), 'scylla-jmx'), replace=replace)
select_java = Path(self.get_jmx_dir()) / 'scripts' / 'select-java'

os.makedirs(os.path.join(self.get_bin_dir(), 'symlinks'), exist_ok=exist_ok)
scylla_jmx_file = os.path.join(self.get_bin_dir(), 'symlinks', 'scylla-jmx')
if os.path.exists(scylla_jmx_file) and replace:
os.remove(scylla_jmx_file)
if java_home := os.environ.get('JAVA_HOME'):
# user selecting specific Java
java_exe = Path(java_home) / 'bin' / 'java'
os.symlink(java_exe, scylla_jmx_file)
elif select_java.exists():
# JMX lookup logic
os.symlink(select_java, scylla_jmx_file)
else:
# older scylla versions, just use default java
java_exe = Path('/usr') / 'bin' / 'java'
os.symlink(java_exe, scylla_jmx_file)

parent_dir = os.path.dirname(os.path.realpath(__file__))
resources_bin_dir = os.path.join(parent_dir, 'resources', BIN_DIR)
Expand Down
Loading