⚠️ ON HOLD - update to pydantic 2.x

pyproject.toml

@@ -16,7 +16,8 @@ keywords = ["data", "exploration", "visualization"]
 python = "^3.9"
 pandas = "^1.3.5"
 ipython = ">=7.31.1"
-pydantic = "^1.9"
+pydantic = "^2.0"
+pydantic-settings = "^2.0.3"
 exceptiongroup = "^1.0.4"
 structlog = "^23.2.0"
 # other dataframes to convert to dataresource format

src/dx/filtering.py

@@ -31,7 +31,7 @@ def store_sample_to_history(df: pd.DataFrame, display_id: str, filters: list) ->
 
     sample_time = pd.Timestamp("now").strftime(settings.DATETIME_STRING_FORMAT)
     # convert from FilterTypes to dicts
-    dex_filters = [dex_filter.dict() for dex_filter in filters]
+    dex_filters = [dex_filter.model_dump() for dex_filter in filters]
     sample = {
         "sampling_time": sample_time,
         "filters": dex_filters,

src/dx/formatters/enhanced.py

@@ -3,10 +3,11 @@
 
 from IPython import get_ipython
 from IPython.core.interactiveshell import InteractiveShell
-from pydantic import BaseSettings, Field
+from pydantic import Field
 
 from dx.formatters.main import DEFAULT_IPYTHON_DISPLAY_FORMATTER, DXDisplayFormatter
 from dx.settings import get_settings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 settings = get_settings()
 
@@ -15,17 +16,16 @@ class DXSettings(BaseSettings):
     DX_DISPLAY_MAX_ROWS: int = 50_000
     DX_DISPLAY_MAX_COLUMNS: int = 50
     DX_MAX_STRING_LENGTH: int = 250
-    DX_HTML_TABLE_SCHEMA: bool = Field(True, allow_mutation=False)
-    DX_MEDIA_TYPE: str = Field("application/vnd.dex.v1+json", allow_mutation=False)
+    DX_HTML_TABLE_SCHEMA: bool = Field(True, frozen=True)
+    DX_MEDIA_TYPE: str = Field("application/vnd.dex.v1+json", frozen=True)
 
     DX_FLATTEN_INDEX_VALUES: bool = False
     DX_FLATTEN_COLUMN_VALUES: bool = True
     DX_STRINGIFY_INDEX_VALUES: bool = False
     DX_STRINGIFY_COLUMN_VALUES: bool = True
-
-    class Config:
-        validate_assignment = True  # we need this to enforce `allow_mutation`
-        json_encoders = {type: lambda t: str(t)}
+    # TODO[pydantic]: The following keys were removed: `json_encoders`.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    model_config = SettingsConfigDict(validate_assignment=True, json_encoders={type: lambda t: str(t)})
 
 
 @lru_cache

src/dx/formatters/plain.py

@@ -6,10 +6,11 @@
 import structlog
 from IPython import get_ipython
 from IPython.core.interactiveshell import InteractiveShell
-from pydantic import BaseSettings, Field
+from pydantic import Field
 
 from dx.formatters.main import DEFAULT_IPYTHON_DISPLAY_FORMATTER
 from dx.settings import get_settings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 logger = structlog.get_logger(__name__)
 settings = get_settings()
@@ -22,12 +23,11 @@ class PandasSettings(BaseSettings):
     PANDAS_DISPLAY_MAX_ROWS: int = 60
     PANDAS_DISPLAY_MAX_COLUMNS: int = 20
     PANDAS_MAX_STRING_LENGTH: int = 50
-    PANDAS_HTML_TABLE_SCHEMA: bool = Field(False, allow_mutation=False)
-    PANDAS_MEDIA_TYPE: str = Field("application/vnd.dataresource+json", allow_mutation=False)
-
-    class Config:
-        validate_assignment = True  # we need this to enforce `allow_mutation`
-        json_encoders = {type: lambda t: str(t)}
+    PANDAS_HTML_TABLE_SCHEMA: bool = Field(False, frozen=True)
+    PANDAS_MEDIA_TYPE: str = Field("application/vnd.dataresource+json", frozen=True)
+    # TODO[pydantic]: The following keys were removed: `json_encoders`.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    model_config = SettingsConfigDict(validate_assignment=True, json_encoders={type: lambda t: str(t)})
 
 
 @lru_cache

src/dx/formatters/simple.py

@@ -3,10 +3,11 @@
 
 from IPython import get_ipython
 from IPython.core.interactiveshell import InteractiveShell
-from pydantic import BaseSettings, Field
+from pydantic import Field
 
 from dx.formatters.main import DEFAULT_IPYTHON_DISPLAY_FORMATTER, DXDisplayFormatter
 from dx.settings import get_settings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 settings = get_settings()
 
@@ -16,17 +17,16 @@ class DataResourceSettings(BaseSettings):
     DATARESOURCE_DISPLAY_MAX_ROWS: int = 50_000
     DATARESOURCE_DISPLAY_MAX_COLUMNS: int = 50
     DATARESOURCE_MAX_STRING_LENGTH: int = 250
-    DATARESOURCE_HTML_TABLE_SCHEMA: bool = Field(True, allow_mutation=False)
-    DATARESOURCE_MEDIA_TYPE: str = Field("application/vnd.dataresource+json", allow_mutation=False)
+    DATARESOURCE_HTML_TABLE_SCHEMA: bool = Field(True, frozen=True)
+    DATARESOURCE_MEDIA_TYPE: str = Field("application/vnd.dataresource+json", frozen=True)
 
     DATARESOURCE_FLATTEN_INDEX_VALUES: bool = False
     DATARESOURCE_FLATTEN_COLUMN_VALUES: bool = True
     DATARESOURCE_STRINGIFY_INDEX_VALUES: bool = False
     DATARESOURCE_STRINGIFY_COLUMN_VALUES: bool = True
-
-    class Config:
-        validate_assignment = True  # we need this to enforce `allow_mutation`
-        json_encoders = {type: lambda t: str(t)}
+    # TODO[pydantic]: The following keys were removed: `json_encoders`.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    model_config = SettingsConfigDict(validate_assignment=True, json_encoders={type: lambda t: str(t)})
 
 
 @lru_cache

src/dx/settings.py

@@ -7,7 +7,8 @@
 import structlog
 from IPython.core.interactiveshell import InteractiveShell
 from pandas import set_option as pandas_set_option
-from pydantic import BaseSettings, validator
+from pydantic import validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from dx.dependencies import get_default_renderable_types
 from dx.types.main import DXDisplayMode, DXSamplingMethod
@@ -61,6 +62,8 @@ class Settings(BaseSettings):
     NUM_PAST_SAMPLES_TRACKED: int = 3
     DB_LOCATION: str = ":memory:"
 
+    # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information.
     @validator("RENDERABLE_TYPES", pre=True, always=True)
     def validate_renderables(cls, vals):
         """Allow passing comma-separated strings or actual types."""
@@ -82,6 +85,8 @@ def validate_renderables(cls, vals):
 
         return valid_vals
 
+    # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information.
     @validator("DISPLAY_MAX_COLUMNS", pre=True, always=True)
     def validate_display_max_columns(cls, val):
         if val < 0:
@@ -91,18 +96,24 @@ def validate_display_max_columns(cls, val):
         pd.set_option("display.max_columns", val)
         return val
 
+    # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information.
     @validator("DISPLAY_MAX_ROWS", pre=True, always=True)
     def validate_display_max_rows(cls, val):
         if val < 0:
             raise ValueError("DISPLAY_MAX_ROWS must be >= 0")
         pd.set_option("display.max_rows", val)
         return val
 
+    # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information.
     @validator("HTML_TABLE_SCHEMA", pre=True, always=True)
     def validate_html_table_schema(cls, val):
         pd.set_option("html.table_schema", val)
         return val
 
+    # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information.
     @validator("MAX_STRING_LENGTH", pre=True, always=True)
     def validate_max_string_length(cls, val):
         if val < 0:
@@ -116,9 +127,7 @@ def get_renderable_types(self):
             **get_default_renderable_types(),
         }
 
-    class Config:
-        validate_assignment = True
-        use_enum_values = True
+    model_config = SettingsConfigDict(validate_assignment=True, use_enum_values=True)
 
 
 @lru_cache
@@ -215,7 +224,7 @@ def set_option(
 @contextmanager
 def settings_context(ipython_shell: Optional[InteractiveShell] = None, **option_kwargs):
     settings = get_settings()
-    orig_settings = settings.dict()
+    orig_settings = settings.model_dump()
     option_kwargs = {str(k).upper(): v for k, v in option_kwargs.items()}
 
     # handle DISPLAY_MODE updates first since it can overwrite other settings

src/dx/utils/formatting.py

@@ -322,7 +322,7 @@ def generate_metadata(
         # these are set whenever store_sample_to_history() is called after a filter action from the frontend
         sample_history = existing_metadata.get("datalink", {}).get("sample_history", [])
         # we shouldn't have a mix of pydantic FilterTypes and dicts here, but just in case
-        filters = [f.dict() if not isinstance(f, dict) else f for f in parent_dxdf.filters]
+        filters = [f.model_dump() if not isinstance(f, dict) else f for f in parent_dxdf.filters]
 
     # TODO: wrap this up into the DXDataFrame init properties with some refactoring
     user_variable_name = variable_name