Merge pull request #1 from PiotrIw/code_update

v1.0.01: code update

.github/workflows/daemon_image_buil_and_push.yml

@@ -0,0 +1,60 @@
+---
+# yamllint disable rule:line-length
+name: Docker Build and Push
+
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    # yamllint disable rule:line-length
+    if: github.event.pull_request.merged == true && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev')
+    # yamllint enable rule:line-length
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Get version from script
+        id: get_version
+        working-directory: ./
+        run: |
+          echo "version=$(python version.py)" >> $GITHUB_ENV
+
+      - name: Get deployment env
+        id: get_deployment
+        run: |
+          if [[ $GITHUB_REF == *"master"* ]]; then
+            DEPLOYMENT="prod"
+          elif [[ $GITHUB_REF == *"dev"* ]]; then
+            DEPLOYMENT="staging"
+          else
+            DEPLOYMENT=${GITHUB_REF#refs/heads/}
+          fi
+          echo "deployment=$DEPLOYMENT" >> $GITHUB_ENV
+          env
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PAT }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.production
+          push: true
+          # yamllint disable rule:line-length
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/${{ vars.REPO_NAME }}:latest-${{ env.deployment }}
+            ${{ secrets.DOCKER_USERNAME }}/${{ vars.REPO_NAME }}:${{ env.version }}-${{ env.deployment }}

.github/workflows/python-app.yml

@@ -1,28 +1,25 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
+---
 name: Python application
 
 on:
   push:
     branches:
       - master
+      - dev
   pull_request:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
-
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.6"
-    - name: Install package manger
-      run: pip install -U pip wheel
-    - name: Install dependencies
-      run: pip install -r requirements.txt
-    - name: Test with pytest
-      run: python test.py
+      - uses: actions/checkout@v3
+      - name: Pull python image
+        run: docker pull python:3.10-slim
+      - name: Build app container
+        run: docker compose build daemon
+      - name: Run app container
+        run: docker compose up -d daemon
+      - name: Run pre-commit hooks
+        uses: pre-commit/[email protected]
+      - name: Run tests
+        run: docker compose run daemon python test.py

.pre-commit-config.yaml

@@ -5,36 +5,36 @@ default_language_version:
   python: python3
 minimum_pre_commit_version: "1.20.0"
 repos:
-  - repo: https://github.com/psf/black
-    rev: 21.9b0
-    hooks:
-    - id: black
-      args:
-        - --config
-        - ./pyproject.toml
-  - repo: https://github.com/PyCQA/flake8
-    rev: 3.9.2
-    hooks:
-    - id: flake8
-      args:
-      - --config=./setup.cfg
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.28.0
+    rev: v3.15.2
     hooks:
-    - id: pyupgrade
-      args:
-      - --py3-only
+      - id: pyupgrade
+        args:
+          - --py3-only
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.13.2
     hooks:
       - id: isort
-        name: isort (python)
         args: ["--profile", "black", "--filter-files"]
   - repo: https://github.com/sirosen/check-jsonschema
-    rev: 0.9.1
+    rev: 0.28.0
     hooks:
       - id: check-github-workflows
-  - repo:  https://github.com/adrienverge/yamllint
-    rev: v1.26.3
-    hooks: 
+      - id: check-readthedocs
+  - repo: https://github.com/adrienverge/yamllint
+    rev: v1.35.1
+    hooks:
       - id: yamllint
+  - repo: https://github.com/psf/black
+    rev: 24.3.0
+    hooks:
+      - id: black
+        args:
+          - --config
+          - ./pyproject.toml
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+        args:
+          - --config=./setup.cfg

.pyup.yml

@@ -1,4 +1,5 @@
-# autogenerated pyup.io config file 
+---
+# autogenerated pyup.io config file
 # see https://pyup.io/docs/configuration/ for all available options
 
 schedule: every month

Dockerfile

@@ -1,13 +1,16 @@
-FROM python:3.9-slim as build
+FROM python:3.10-slim as build
 LABEL maintainer="adam.dobrawy{at}siecobywatelska.pl"
 
 WORKDIR /usr/src/app
 COPY requirements.txt ./
+COPY requirements.dev.txt ./
 RUN apt-get update \
- && apt-get install -y --no-install-recommends g++ \
+ && apt-get install -y --no-install-recommends g++ git make nano \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/* \
+ && pip install --no-cache-dir --upgrade pip \
  && pip install --no-cache-dir -r requirements.txt \
+ && pip install --no-cache-dir -r requirements.dev.txt \
  && apt-get remove -y g++ && apt-get autoremove -y
 ENV PYTHONUNBUFFERED=1
 COPY . .
@@ -18,4 +21,6 @@ RUN python test.py
 
 # Actual image
 FROM build
-CMD [ "python", "./daemon.py" ]
+# CMD [ "python", "./daemon.py" ]
+# Chended to allow VScode run and debug app or debug container starting errors:
+CMD [ "bash", "-c", "echo 'imap-to-webhook container started' ;  sleep infinity " ]

Dockerfile.production

@@ -0,0 +1,15 @@
+FROM python:3.10-slim as build
+LABEL maintainer="adam.dobrawy{at}siecobywatelska.pl"
+
+WORKDIR /usr/src/app
+COPY requirements.txt ./
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends g++ git make nano \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip install --no-cache-dir --upgrade pip \
+ && pip install --no-cache-dir -r requirements.txt \
+ && apt-get remove -y g++ && apt-get autoremove -y
+ENV PYTHONUNBUFFERED=1
+COPY . .
+CMD [ "python", "./daemon.py" ]

Makefile

@@ -0,0 +1,23 @@
+start:
+	docker-compose up
+
+stop:
+	docker-compose stop
+
+clean:
+	docker-compose down
+
+build:
+	docker-compose build
+
+test:
+	docker-compose run --rm daemon python test.py
+
+lint: # lint currently staged files
+	pre-commit run
+
+lint-all: # lint all files in repository
+	pre-commit run --all-files
+
+code-syle-check:
+	docker-compose run daemon pre-commit run --all-files

connection.py

@@ -4,14 +4,14 @@ def __init__(self, config):
         hostname = config["imap"]["hostname"]
         port = config["imap"]["port"]
         self.client = transport(host=hostname, port=port)
-        print("Connected to mail server")
         username = config["imap"]["username"]
         password = config["imap"]["password"]
+        print(f"Connecting to mail server: {hostname}")
         if username and password:
             login = self.client.login(username, password)
             if login[0] != "OK":
                 raise Exception("Unable to login", login)
-        print("Logged in")
+        print(f"Logged in as {username}")
         select_folder = self.client.select(config["imap"]["inbox"])
         if select_folder[0] != "OK":
             raise Exception("Unable to select folder", select_folder)

daemon.py

@@ -1,17 +1,28 @@
+import copy
 import os
 import time
-from parser import serialize_mail
 
 import requests
 import sentry_sdk
 
 from config import get_config
 from connection import IMAPClient
+from mail_parser import serialize_mail
+
+__version__ = "1.0.01"
+
 
 def main():
     config = get_config(os.environ)
+    config_printout = copy.deepcopy(config)
+    if "password" in config_printout.get("imap", {}):
+        config_printout["imap"]["password"] = "********"
+        print("Configuration to print: ", config_printout)
+        print("Configuration: ", config)
+
     session = requests.Session()
-    print("Configuration: ", config)
+    print(f"Starting daemon version {__version__}")
+    print("Configuration: ", config_printout)
     sentry_sdk.init(dsn=config["sentry_dsn"], traces_sample_rate=1.0)
     if config["sentry_dsn"]:
         try:
@@ -51,7 +62,7 @@ def process_msg(client, msg_id, config, session, sentry_client=None):
         body = serialize_mail(raw_mail, config["compress_eml"])
         end = time.time()
         print("Message serialized in {} seconds".format(end - start))
-        res = session.post(config['webhook'], files=body)
+        res = session.post(config["webhook"], files=body)
         print("Received response:", res.text)
         res.raise_for_status()
         response = res.json()

docker-compose.yml

@@ -8,6 +8,8 @@ services:
       IMAP_URL: ${IMAP_URL}
       WEBHOOK_URL: http://mock:5000/
       ON_SUCCESSS: 'delete'
+    volumes:
+      - .:/usr/src/app
 
   mock:
     build:

parser.py → mail_parser.py

@@ -26,6 +26,7 @@
 EML_MIME = "message/rfc822"
 BINARY_MIME = "application/octet-stream"
 
+
 def get_text(mail):
     raw_content, html_content, plain_content, html_quote, plain_quote = (
         "",
@@ -46,7 +47,8 @@ def get_text(mail):
         plain_content = talon.quotations.extract_from_plain(raw_content)
         plain_quote = raw_content.replace(plain_content, "")
 
-    # 'content' item holds plain_content and 'quote' item holds plain_quote (with HTML stripped off).
+    # 'content' item holds plain_content and 'quote' item holds plain_quote
+    # (with HTML stripped off).
     # These names are used for backward compatibility.
     return {
         "html_content": html_content,
@@ -168,9 +170,10 @@ def serialize_mail(raw_mail, compress_eml=False):
 
 
 if __name__ == "__main__":
-    import sys, pprint
-    with open(sys.argv[1], 'rb') as fp:
+    import sys
+
+    with open(sys.argv[1], "rb") as fp:
         raw_mail = fp.read()
         mail = mailparser.parse_from_bytes(raw_mail)
         body = get_manifest(mail, False)
-        json.dump(body, sys.stdout, indent=4)
+        json.dump(body, sys.stdout, indent=4)

mock/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3
+FROM  python:3.10-slim
 LABEL maintainer="adam.dobrawy{at}siecobywatelska.pl"
 
 WORKDIR /usr/src/app
@@ -10,4 +10,4 @@ COPY . .
 
 ENV FLASK_APP=mock.py
 
-CMD [ "flask" , "run", "-h", "0.0.0.0" ]
+CMD [ "flask" , "run", "-h", "0.0.0.0" ]

mock/requirements.txt

@@ -1 +1 @@
-Flask==1.1.1
+Flask==3.0.2

pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.black]
 line-length = 88
-target-version = ['py36', 'py37', 'py38']
+target-version = ['py310']
 exclude = '''
 /(
     node_modules

requirements.dev.txt

@@ -1 +1 @@
-pre-commit==1.21.0
+pre-commit==3.7.0

requirements.txt

@@ -1,6 +1,9 @@
-requests==2.22.0
-mail-parser==3.9.3
-html2text==2019.8.11
+requests==2.31.0
+mail-parser==3.15.0
+html2text==2024.2.26
+# pypi talon to be replaced - last update 2017
 talon==1.4.4
-ipdb==0.13.9
-sentry-sdk==1.5.1
+# forked talon - scikit version does not match training data
+# git+https://github.com/PiotrIw/talon.git@master
+ipdb==0.13.13
+sentry-sdk==1.43.0

test.py

@@ -1,7 +1,8 @@
 import json
 import os
 import unittest
-from parser import serialize_mail
+
+from mail_parser import serialize_mail
 
 
 def get_email_as_bytes(name):

version.py

@@ -0,0 +1,4 @@
+from daemon import __version__
+
+if __name__ == "__main__":
+    print(f"v{__version__}")