Merge remote-tracking branch 'origin/develop' into 199-add-faiss-hnsw

.github/coverage/cpp.develop.coverage_report.txt

@@ -11,7 +11,7 @@ src/BackendNeo4j.cc                          121       0     0%   4,6-17,20,24,2
 src/BlobCommand.cc                            87      66    75%   76,130-132,136-139,145,147,165,186-189,192-196,202
 src/BoundingBoxCommand.cc                    180       4     2%   45,49,51,53-54,56-59,62,64-67,70-73,76,83,87,90-91,93-97,101,103,105,114,118,122-123,125-132,137-138,140-144,147-150,152,154-160,162-165,167-169,171-173,176-177,179-181,183-184,186-187,190,193,196-197,199,201-204,206-210,213,215-219,222-223,225-227,229-237,240-244,246,251-256,259-261,263,265-266,268,270,272-274,276-277,281-283,286,288,292-294,296,298,300-303,307-308,310-313,316-319,321-326,329-330,335,338-339,341
 src/CommunicationManager.cc                   46       0     0%   42-43,46-47,49-50,52-54,57,61-66,68-71,73-81,84,86-88,93,96-97,100-101,105,107-108,110,113-116
-src/DescriptorsCommand.cc                    602     107    17%   56,63-68,73,75-79,81-85,87,89,92-93,96,98-99,102,104-105,108-114,117,120,123,170-172,176,190-194,234-245,255,269-271,275,290-297,299,311-314,319,324-328,344,352,354-359,362-365,368,371-372,374-378,381,384,386-387,390-392,394-395,397-398,401-402,404-406,412,416,418,420,422-423,426,428-431,437-438,441,443-444,446,448-449,452,455,457,459,462,464-469,471-475,477,480,483-484,487-488,499,502,505,507-509,517,521,523,526,528-531,534-539,541-545,547,550,552,554,557,560-561,563,565,567-572,575,577-579,582-583,585,587,589-590,592,594,596-598,600,602-607,612-613,616,618,620,627-628,631,635,637,640,642-645,648-652,654-658,660,662-665,667-669,672,676-685,690-691,694,701,703,706-707,710,714,720,722,725,728-729,733,738,740,742,745,748-749,751-754,758,762-763,765,767-769,771,773-774,776,778-780,782-784,789-791,794-795,797,800-804,808,811,815,817-818,820-821,823,825-826,828-830,832,837,839-840,842-844,846-847,851,853-856,858-861,866,869-871,873,875,877-880,882-886,889-890,893,895,897,899-903,906-907,910-913,915,917-924,929,931,933-934,937-940,942,944,946-953,957-962,968,971,973,975-978,981,983,986-989,991-995,1001,1003,1005-1008,1013,1015,1017-1018,1021-1023,1025,1027,1029-1032,1035-1036,1038-1039,1041,1043-1044,1046-1052,1056-1057,1061-1062,1064-1065,1073-1074,1077-1078,1080,1082-1089,1093,1097-1098,1102-1106,1111-1112,1114
+src/DescriptorsCommand.cc                    668     107    16%   56,63-68,73,75-79,81-85,87,89,92-93,96-98,101,103-104,107-113,115,118,121,168-170,174,188-192,232-243,253,267-269,273,288-295,297,309-312,317,322-326,343,351,353-356,359,362-363,365-369,372,375,377-378,381-383,385-386,388-389,392-393,395-397,403,408-409,411,413-414,417,419-422,425-430,436-437,440,442-443,445,447-448,451,454,456,458,461,463-468,470-474,476,479,482-483,486-487,498,501,506,512-513,518-519,521-524,527,533-534,539-540,543-549,552,554-555,557,559-560,562-563,567,569-574,577-582,585,588-593,595,621-622,625-626,629,632,639-640,642-644,646,649-650,652,655,658,660-662,670,674,676,679,681-684,687-692,694-698,700,703,705,707,710,713-714,716,718,720-725,728,730-732,735-736,738,740,742-743,745,747,749-751,753,755-760,765-766,769,771,773,780-781,784,788,790,793,795-798,801-805,807-811,813,815-818,820-822,825,829-838,843-844,847,853,855,858-859,862,866,871,873,876,879-880,884,889,891,893,896,899-900,902-905,909,913-914,916,918-920,922,924-925,927,929-931,933-935,940-942,945-946,948,951-955,959,962,966,968-969,971-972,974,976-977,979-981,983,988,990-991,993-995,997-998,1002,1004-1007,1009-1012,1017,1020-1022,1024,1026,1028-1031,1033-1037,1040-1041,1044,1046,1048,1050-1054,1057-1058,1061-1064,1066,1068-1075,1080,1082,1084-1085,1088-1091,1093,1095,1097-1104,1108-1113,1119,1122,1124,1126-1129,1132,1134,1137-1140,1142-1146,1152,1154,1156-1159,1164,1166,1168-1169,1172-1174,1176,1178,1180-1183,1186-1187,1189-1190,1192,1194-1195,1197-1203,1207-1208,1212-1213,1215-1216,1224-1225,1228-1229,1231,1233-1240,1244,1248-1249,1253-1257,1262-1263,1265
 src/DescriptorsManager.cc                     24      19    79%   49-50,57-58,73
 src/ExceptionsCommand.cc                       6       0     0%   35-40
 src/ImageCommand.cc                          322     157    48%   55,59,63,65,67-69,71,73-76,78,81,86,88-89,97,99,106,109,111-112,114-115,117-118,120-121,124,151,162-163,174-175,177,182-185,195-196,198,203-206,221-229,231-233,246-247,257-267,269-270,272-273,278,286,297,304,308,311,313,315,337,339-340,343-348,350,352,374-376,379-381,385-388,394,396,403-406,420,427,433-436,440-441,452-455,458-463,468-470,481-484,489-493,498-499,501-502,504-508,511,513-517,520-523,526-527,530,532,537
@@ -25,7 +25,7 @@ src/PMGDQueryHandler.cc                      623     517    83%   82-84,166-167,
 src/QueryHandlerBase.cc                       32       6    18%   26-29,35-36,38,42-43,45,47-48,52,56-58,60-62,64-66,68-69,72-73
 src/QueryHandlerExample.cc                    33      18    54%   65-67,75-78,84-85,89-92,94-95
 src/QueryHandlerNeo4j.cc                     139       0     0%   53,55-56,58,60-62,64-65,67,70-76,80-81,83-87,91,93,95-100,104-108,111-115,119-126,129-132,134-136,139-147,149-153,159,162,169,172-175,177-179,181,184-187,189-190,192,194,197,199,201-204,207-208,210,212-213,216,218,222-223,225,229,231-232,235,237-240,243-247,250-254,256-257,261-268,271-274,277
-src/QueryHandlerPMGD.cc                      344     226    65%   109-111,119-122,137-138,142-146,149-153,157-164,167-170,172-174,183-185,189-191,209-211,216-218,233-239,243-245,262,264-273,302-304,350-352,354-356,359-360,362-365,388-389,391-392,401,407-419,421-423,430-438,475,477,532-533,535-536
+src/QueryHandlerPMGD.cc                      344     226    65%   109-111,119-122,137-138,142-146,149-153,157-164,167-170,172-174,183-185,189-191,209-211,216-218,233-239,243-245,263,265-274,303-305,352-354,356-358,361-362,364-367,390-391,393-394,403,410-422,424-426,433-441,478,480,535-536,538-539
 src/QueryMessage.cc                           12       0     0%   37-39,42-43,45-46,48,51-54
 src/RSCommand.cc                             144     105    72%   65-67,73-74,98,100-101,103,110,131,134-138,141,172-174,176,178-181,188,262,285,287-289,291-297,301
 src/SearchExpression.cc                       99      38    38%   59,132-133,135,137-139,143,146,148-153,157,160,168-170,177,180-181,183-185,188,192-195,197,201,217-222,224-225,227,235-240,243,247-249,252-256,263,276,284-285
@@ -57,5 +57,5 @@ utils/src/comm/Exception.cc                    6       0     0%   35-40
 utils/src/stats/SystemStats.cc               250     249    99%   453
 utils/src/timers/TimerMap.cc                  82      75    91%   126,151,153,155-158
 ------------------------------------------------------------------------------
-TOTAL                                      10076    6488    64%
+TOTAL                                      10142    6488    64%
 ------------------------------------------------------------------------------

.github/coverage/cpp.develop.coverage_value.txt

@@ -1 +1 @@
-64.3906
+63.9716

.github/coverage/python.develop.coverage_report.txt

@@ -1,6 +1,6 @@
 Name                                   Stmts   Miss  Cover   Missing
 --------------------------------------------------------------------
 /vdms/client/python/vdms/__init__.py       2      0   100%
-/vdms/client/python/vdms/vdms.py          98      2    98%   151, 166
+/vdms/client/python/vdms/vdms.py          99      2    98%   152, 167
 --------------------------------------------------------------------
-TOTAL                                    100      2    98%
+TOTAL                                    101      2    98%

.github/coverage/python.develop.coverage_value.txt

@@ -1 +1 @@
-98
+98.02

.github/requirements.txt

@@ -3,7 +3,7 @@ cffi==1.16.0
 click==8.1.7
 colorlog==6.8.2
 coverage==7.6.0
-cryptography==42.0.8
+cryptography==43.0.1
 Flask==3.0.3
 gcovr==7.2
 imutils==0.5.4

.github/workflows/CI.yml

@@ -55,9 +55,10 @@ jobs:
   Update:
     # name: Lint & Update Reports
     needs: [Testing, Results]
-    if: ${{ always() && (needs.Results.result == 'success') }}
+    if: ${{ always() }} 
     uses: ./.github/workflows/_CI_update.yml
     with:
         coverage_value_updated: ${{ needs.Testing.outputs.coverage_value_updated }}
+        coverage_test_status: ${{ needs.Testing.outputs.coverage_test_status }}
     secrets: inherit
 

.github/workflows/_CI_coverage.yml

@@ -103,6 +103,7 @@ jobs:
       source_old_cpp_value: ${{ steps.report_coverage.outputs.source_old_cpp_value }}
       source_old_py_value: ${{ steps.report_coverage.outputs.source_old_py_value }}
       coverage_value_updated: ${{ steps.report_coverage.outputs.cov_changed }}
+      coverage_test_status: ${{ steps.report_coverage.outputs.coverage_test_status }}
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
         with:
@@ -236,6 +237,7 @@ jobs:
           set -x
 
           did_cov_change='false'
+          test_status='failed'
           if [ "$pr_dev_value_cpp" != "$coverage_value_cpp" ]; then
             did_cov_change='true'
           fi
@@ -262,10 +264,13 @@ jobs:
           then
             exit 1
           fi
+
+          test_status='passed'
           echo "Source Python Coverage: ${coverage_value_py}"
           echo "source_coverage_py=${coverage_value_py}" >> $GITHUB_OUTPUT
           echo "source_old_py_value=${pr_dev_value_py}" >> $GITHUB_OUTPUT
           # echo "target_coverage_py=${target_value_py}" >> $GITHUB_OUTPUT
+          echo "coverage_test_status=${test_status}" >> $GITHUB_OUTPUT
 
       - name: Upload New coverage results
         uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b  # v4.3.4

.github/workflows/_CI_update.yml

@@ -7,6 +7,10 @@ on:
           required: true
           description: "The C++ Coverage for target"
           type: string
+        coverage_test_status:
+          required: true
+          description: "Status of coverage tests (passed/failed)"
+          type: string
 
 permissions: write-all
 
@@ -19,7 +23,6 @@ jobs:
         runs-on: ubuntu-latest
         steps:
           # Checkout code doesn't persist across jobs
-          # If formatting needed, checkout and format again
           - name: Checkout Source Branch
             uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
             with:
@@ -31,7 +34,7 @@ jobs:
           - run: mkdir -p ${{ env.DOCKER_ARTIFACT_DIR }}
 
           - name: Retrieve Current Coverage Files
-            if: ${{ inputs.coverage_value_updated }} == 'true'
+            if: ${{ inputs.coverage_value_updated }} == 'true' && ${{ inputs.coverage_test_status }} == 'passed'
             uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
             with:
               name: coverage_artifact
@@ -43,7 +46,7 @@ jobs:
 
           - name: Update coverage reports with latest coverage
             # Change latest coverage as develop (future target)
-            if: ${{ inputs.coverage_value_updated }} == 'true'
+            if: ${{ inputs.coverage_value_updated }} == 'true' && ${{ inputs.coverage_test_status }} == 'passed'
             run: |
               cd ${GITHUB_WORKSPACE}/.github/coverage/
               rm -rf *.develop.*.txt || true

client/python/vdms/vdms.py

@@ -85,6 +85,7 @@ def connect(self, host="localhost", port=55555):
 
             if self.use_tls:
                 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
+                context.minimum_version = ssl.TLSVersion.TLSv1_2
                 if self.ca_file != "":
                     context.load_verify_locations(cafile=self.ca_file)
                 if self.cert_file != "" and self.key_file != "":

remote_function/udf_server.py

@@ -1,15 +1,12 @@
 from flask import Flask, request, jsonify, send_file, after_this_request
 import cv2
-import numpy as np
 import json
 from datetime import datetime, timezone
 import os
 import sys
-from collections import defaultdict, deque
-import skvideo.io
-import imutils
 import uuid
 from zipfile import ZipFile
+from werkzeug.utils import secure_filename
 
 for entry in os.scandir("functions"):
     if entry.is_file():
@@ -18,8 +15,6 @@
 
 app = Flask(__name__)
 
-count = 0
-
 
 def get_current_timestamp():
     dt = datetime.now(timezone.utc)
@@ -42,7 +37,7 @@ def image_api():
 
     format = json_data["format"] if "format" in json_data else "jpg"
 
-    tmpfile = "tmpfile" + uuid.uuid1().hex + "." + str(format)
+    tmpfile = secure_filename("tmpfile" + uuid.uuid1().hex + "." + str(format))
 
     image_data.save(tmpfile)
 
@@ -70,7 +65,7 @@ def video_api():
     video_data = request.files["videoData"]
     format = json_data["format"] if "format" in json_data else "mp4"
 
-    tmpfile = "tmpfile" + uuid.uuid1().hex + "." + str(format)
+    tmpfile = secure_filename("tmpfile" + uuid.uuid1().hex + "." + str(format))
     video_data.save(tmpfile)
 
     video_file, metadata_file = "", ""