Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

⚡ Remove monkeypatch on pandas del method #178

Merged
merged 4 commits into from
Sep 5, 2024
Merged

⚡ Remove monkeypatch on pandas del method #178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6e15c57
:zap: Remove monkeypatch on pandas del method
HugoPerrier Sep 5, 2024
49341e1
:zap: Remove monkeypatch on pandas del method
HugoPerrier Sep 5, 2024
ceba992
:zap: Avoid f-string when logging
HugoPerrier Sep 5, 2024
020489a
:zap: Avoid f-string when logging
HugoPerrier Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 0 additions & 37 deletions melusine/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,6 @@
"""

import logging
from ctypes import CDLL, cdll
from typing import Any, Optional

import pandas as pd

from melusine._config import config

Expand All @@ -19,36 +15,3 @@
# LOGGING
# ------------------------------- #
logging.getLogger(__name__).addHandler(logging.NullHandler())


# ------------------------------- #
# MONKEY PATCH
# ------------------------------- #

# Monkey patch for pandas DataFrame memory leaking on linux OS (pandas issue #2659)
try:
# Try executing linux malloc_trim function (release free memory)
cdll.LoadLibrary("libc.so.6")
libc: Optional[CDLL] = CDLL("libc.so.6")
if libc is not None:
libc.malloc_trim(0)
except (OSError, AttributeError): # pragma: no cover
# Incompatible OS: this monkey patch is not needed
libc = None

# Store the standard pandas method
__std_del: Optional[Any] = getattr(pd.DataFrame, "__del__", None)


# Prepare a new __del__ method
def __fixed_del(self: Any) -> None: # pragma: no cover
"""Override DataFrame's __del__ method: call the standard method + release free memory with malloc_trim."""
if __std_del is not None:
__std_del(self)
if libc is not None:
libc.malloc_trim(0)


# Override standard pandas method if needed
if libc is not None:
pd.DataFrame.__del__ = __fixed_del
2 changes: 1 addition & 1 deletion melusine/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ def transform(self, df: MelusineDataset) -> MelusineDataset:
return_cols.append(self.debug_dict_col)

for method in self.transform_methods:
logger.debug(f"Running transform for {type(self).__name__} ({method.__name__})")
logger.debug("Running transform for %s (%s)", type(self).__name__, getattr(method, "__name__", "unnamed"))
first_arg_name: str = list(inspect.signature(method).parameters)[0]

if first_arg_name == "row":
Expand Down
Loading