Add precommit hook

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -61,4 +61,4 @@ body:
 - type: markdown
   attributes:
     value: >
-      Thanks for helping us improve the library!
+      Thanks for helping us improve the library!

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -31,4 +31,4 @@ body:
 - type: markdown
   attributes:
     value: >
-      Thanks for contributing 🎉
+      Thanks for contributing 🎉

.github/collect_env.py

@@ -1,7 +1,7 @@
 # Copyright (C) 2020-2022, Pyronear.
 
 # This program is licensed under the Apache License 2.0.
-# See LICENSE or go to <https://www.apache.org/licenses/LICENSE-2.0> for full license details.
+# See LICENSE or go to <https://www.apache.org/licenses/LICENSE-2.0> for full license.
 
 """
 Based on https://github.com/pytorch/pytorch/blob/master/torch/utils/collect_env.py

.github/workflows/doc-status.yml

@@ -19,4 +19,4 @@ jobs:
         shell: python
         env:
           STATUS: ${{ github.event.build.status }}
-          ERROR: ${{ github.event.build.error.message }}
+          ERROR: ${{ github.event.build.error.message }}

.github/workflows/gh-page.yml

@@ -19,4 +19,4 @@ jobs:
         shell: python
         env:
           STATUS: ${{ github.event.build.status }}
-          ERROR: ${{ github.event.build.error.message }}
+          ERROR: ${{ github.event.build.error.message }}

.gitignore

@@ -118,3 +118,6 @@ pyroengine/version.py
 *.onnx
 # Release
 conda-dist/
+
+docker-compose.yml.bak
+docker-compose.override.yml

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+default_language_version:
+    python: python3.9
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+        exclude: .conda
+      - id: check-toml
+      - id: check-added-large-files
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-ast
+      - id: check-json
+      - id: check-merge-conflict
+      - id: no-commit-to-branch
+        args: ['--branch', 'main']
+      - id: debug-statements
+        language_version: python3
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.3.0
+    hooks:
+      - id: black
+        language_version: python3.9
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.290'
+    hooks:
+      - id: ruff
+        args:
+          - --fix
+          - --line-length=120

LICENSE

@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

Makefile

@@ -26,11 +26,13 @@ build:
 
 # Run the engine wrapper
 run:
+	bash scripts/setup-docker-compose.sh
 	docker build . -t pyronear/pyro-engine:latest
 	docker compose up -d
+	rm docker-compose.yml.bak
 
 # Get log from engine wrapper
-log: 
+log:
 	docker logs -f --tail 50 pyro-engine-run
 
 # Stop the engine wrapper

README.md

@@ -48,7 +48,7 @@ engine = Engine()
 
 im = Image.open("path/to/your/image.jpg").convert('RGB')
 
-prediction = engine.predict(image) 
+prediction = engine.predict(image)
 ```
 
 ## Setup

docker-compose.yml

@@ -12,6 +12,11 @@ services:
     volumes:
       - ./data:/usr/src/app/data
     restart: always
+    deploy:
+      resources:
+        limits:
+          cpus: "3"
+          memory: ""
     logging:
       driver: "json-file"
       options:

docs/source/conf.py

@@ -19,9 +19,10 @@
 import sys
 from datetime import datetime
 
-sys.path.insert(0, os.path.abspath("../.."))
 import pyroengine
 
+sys.path.insert(0, os.path.abspath("../.."))
+
 # -- Project information -----------------------------------------------------
 
 master_doc = "index"
@@ -86,7 +87,7 @@
                 <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 16 16">
                     <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path>
                 </svg>
-            """,
+            """,  # noqa: E501
             "class": "",
         },
     ],

docs/source/engine.rst

@@ -12,4 +12,4 @@ by predicting whether there is a fire on this image
 
 	.. automethod:: clear_cache
 	.. automethod:: predict
-	.. automethod:: heartbeat
+	.. automethod:: heartbeat

pyproject.toml

@@ -36,7 +36,7 @@ dependencies = [
     "pyroclient @ git+https://github.com/pyronear/pyro-api.git@main#egg=pkg&subdirectory=client",
     "requests>=2.20.0,<3.0.0",
     "opencv-python==4.5.5.64",
-    "tqdm>=4.62.0", 
+    "tqdm>=4.62.0",
 ]
 
 [project.optional-dependencies]

pyroengine/__init__.py

@@ -1,3 +1,3 @@
-from .core import *
-from . import engine, sensors, utils
-from .version import __version__
+from .core import *  # noqa: F403
+from . import engine, sensors, utils  # noqa
+from .version import __version__  # noqa

pyroengine/core.py

@@ -42,8 +42,9 @@
             img = self.cameras[idx].capture()
             try:
                 self.engine.predict(img, self.cameras[idx].ip_address)
-            except Exception:
+            except Exception as e:
                 logging.warning(f"Unable to analyze stream from camera {self.cameras[idx]}")
+                logging.warning(e)
         except Exception:
             logging.warning(f"Unable to fetch stream from camera {self.cameras[idx]}")
 

pyroengine/engine.py

@@ -141,7 +141,11 @@ def __init__(
 
         # Var initialization
         self._states: Dict[str, Dict[str, Any]] = {
-            "-1": {"last_predictions": deque([], self.nb_consecutive_frames), "frame_count": 0, "ongoing": False},
+            "-1": {
+                "last_predictions": deque([], self.nb_consecutive_frames),
+                "frame_count": 0,
+                "ongoing": False,
+            },
         }
         if isinstance(cam_creds, dict):
             for cam_id in cam_creds:
@@ -245,7 +249,6 @@ def _update_states(self, frame: Image.Image, preds: np.ndarray, cam_key: str) ->
             conf = np.max(best_boxes_scores) / (self.nb_consecutive_frames + 1)  # memory + preds
 
             if len(combine_predictions):
-
                 # send only preds boxes that match combine_predictions
                 ious = box_iou(combine_predictions[:, :4], preds[:, :4])
                 iou_match = [np.max(iou) > 0 for iou in ious]
@@ -256,7 +259,13 @@ def _update_states(self, frame: Image.Image, preds: np.ndarray, cam_key: str) ->
                 output_predictions = output_predictions[:5, :]  # max 5 bbox
 
         self._states[cam_key]["last_predictions"].append(
-            (frame, preds, output_predictions.tolist(), datetime.utcnow().isoformat(), False)
+            (
+                frame,
+                preds,
+                output_predictions.tolist(),
+                datetime.utcnow().isoformat(),
+                False,
+            )
         )
 
         # update state
@@ -309,7 +318,13 @@ def predict(self, frame: Image.Image, cam_id: Optional[str] = None) -> float:
                 ):
                     if not is_staged:
                         self._stage_alert(frame, cam_id, ts, localization)
-                        self._states[cam_key]["last_predictions"][idx] = frame, preds, localization, ts, True
+                        self._states[cam_key]["last_predictions"][idx] = (
+                            frame,
+                            preds,
+                            localization,
+                            ts,
+                            True,
+                        )
 
         else:
             conf = 0  # return default value

scripts/get_sunset_sunrise.sh

@@ -20,4 +20,3 @@ sunset=$(date --date="$SUNS" +%R)
 
 echo $sunrise > /home/pi/pyro-engine/data/sunset_sunrise.txt
 echo $sunset >> /home/pi/pyro-engine/data/sunset_sunrise.txt
-

scripts/setup-docker-compose.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Define the percentage of host memory you want to allocate
+PERCENTAGE=70
+
+# Get the total memory of the host system in kilobytes
+TOTAL_MEM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}')
+
+# Calculate the memory limit in kilobytes
+LIMIT_MEM_KB=$((TOTAL_MEM_KB * PERCENTAGE / 100))
+
+# Convert the limit to a format Docker understands (e.g., "m" for megabytes)
+LIMIT_MEM_MB=$((LIMIT_MEM_KB / 1024))m
+
+# Define the Docker Compose file to modify
+DOCKER_COMPOSE_FILE="docker-compose.yml"
+
+# Backup the original Docker Compose file
+cp $DOCKER_COMPOSE_FILE "${DOCKER_COMPOSE_FILE}.bak"
+
+# Use awk to update the memory limits in the Docker Compose file, preserving indentation
+awk -v mem_limit="$LIMIT_MEM_MB" '
+/services:/ { in_services=1 }
+in_services && /deploy:/ { in_deploy=1 }
+in_deploy && /resources:/ { in_resources=1 }
+in_resources && /limits:/ { in_limits=1 }
+in_limits && /memory:/ {
+    $0 = gensub(/memory:.*/, "memory: " mem_limit, 1)
+}
+{ print }
+' "${DOCKER_COMPOSE_FILE}.bak" > "docker-compose.override.yml"
+
+echo "Memory limits set to $LIMIT_MEM_MB in $DOCKER_COMPOSE_FILE"

scripts/update_script.sh

@@ -20,11 +20,10 @@ UPSTREAMHASH=$(git rev-parse refs/remotes/origin/main)
 # Compare hashes and update if they are different
 if [ "$HEADHASH" != "$UPSTREAMHASH" ]
 then
-    echo "$(date): New changes detected! Updating and executing script..."    
+    echo "$(date): New changes detected! Updating and executing script..."
     git pull origin main
     bash /home/pi/pyro-engine/scripts/debug_script.sh
     echo "$(date): Update done!"
 else
     echo "$(date): No changes detected"
 fi
-

scripts/update_script_develop.sh

@@ -20,11 +20,10 @@ UPSTREAMHASH=$(git rev-parse refs/remotes/origin/develop)
 # Compare hashes and update if they are different
 if [ "$HEADHASH" != "$UPSTREAMHASH" ]
 then
-    echo "$(date): New changes detected! Updating and executing script..."    
+    echo "$(date): New changes detected! Updating and executing script..."
     git pull origin develop
     bash /home/pi/pyro-engine/scripts/debug_script.sh
     echo "$(date): Update done!"
 else
     echo "$(date): No changes detected"
 fi
-