Merge pull request #156 from eth-cscs/dev

Merge to master (version 1.8.5)

CONTRIBUTORS.md

@@ -6,4 +6,5 @@
 - Dorsch, Juan Pablo. ETH Zurich - CSCS
 - Klein, Mark. ETH Zurich - CSCS
 - Koutsaniti, Eirini. ETH Zurich - CSCS
-- Lezcano, Facundo. UNL-CONICET - CIMEC
+- Lezcano, Facundo. UNL-CONICET - CIMEC
+- Sewell, Christopher John. EPFL

deploy/demo/clean_logs.sh

@@ -4,6 +4,6 @@ rm -fr logs/client/*.log
 rm -fr logs/cluster/*.log
 rm -fr logs/firecrest/*.log
 rm -fr logs/keycloak/*.log
-rm -fr minio/*
+rm -fr minio/.minio.sys
 rm -fr taskpersistence-data/dump.rdb taskpersistence-data/redis.log
 echo "done."

doc/openapi/firecrest-developers-api.yaml

@@ -610,6 +610,74 @@ paths:
             text/plain:
               schema:
                 type: string
+  '/utilities/stat':
+    parameters:
+      - in: header
+        name: X-Machine-Name
+        description: The system name
+        required: true
+        schema:
+          type: string
+    get:
+      summary: determines the status of a file
+      description: Uses the stat linux application to determine the status of a file on the
+        {X-Machine-Name} filesystem.
+      tags:
+        - Utilities
+      parameters:
+        - name: targetPath
+          in: query
+          description: Absolute filesystem path
+          required: true
+          allowEmptyValue: false
+          schema:
+            type: string
+          allowReserved: true
+        - name: dereference
+          in: query
+          description: Follow symbolic links
+          schema:
+            type: boolean
+      responses:
+        '200':
+          description: Operation completed
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Application-output'
+        '400':
+          description: Error in file operation
+          content:
+            text/plain:
+              schema:
+                type: string
+          headers:
+            X-Machine-Does-Not-Exist:
+              description: Machine does not exist
+              schema:
+                type: string
+            X-Machine-Not-Available:
+              description: Machine is not available
+              schema:
+                type: string
+            X-Permission-Denied:
+              description: User does not have permissions to access machine or paths
+              schema:
+                type: string
+            X-Invalid-Path:
+              description: <path> is an invalid path
+              schema:
+                type: string
+            X-Timeout:
+              description: Command has finished with timeout signal
+              schema:
+                type: string
+        '401':
+          description: No auth header given
+          content:
+            text/plain:
+              schema:
+                type: string
   '/utilities/symlink':
     parameters:
       - in: header

src/tests/automated_tests/unit/test_unit_utilities.py

@@ -43,6 +43,18 @@
 		 (SERVER_UTILITIES, 400, "(a"),
 		 (SERVER_UTILITIES, 400, "`hostname`") ]
 
+# test data for stat
+DATA_STAT = [ (SERVER_UTILITIES, 200, ".bashrc") ,
+		 (SERVER_UTILITIES, 200, "/var/log/messages") ,
+         ("someservernotavailable", 400, ".bashrc"),
+		 (SERVER_UTILITIES, 400, "nofile") ,
+		 (SERVER_UTILITIES, 400, "/\\") ,
+		 (SERVER_UTILITIES, 400, "d["),
+		 (SERVER_UTILITIES, 400, "a>b"),
+		 (SERVER_UTILITIES, 400, "a<b"), 
+		 (SERVER_UTILITIES, 400, "(a"),
+		 (SERVER_UTILITIES, 400, "`hostname`") ]
+
 # test data for #mkdir, symlink
 DATA_201 = [ (SERVER_UTILITIES, 201) , ("someservernotavailable", 400)]
 
@@ -130,6 +142,19 @@ def test_file_type(machine, expected_response_code, file_name, headers):
 	print(resp.headers)
 	assert resp.status_code == expected_response_code
 
+
+@skipif_not_uses_gateway
+@pytest.mark.parametrize("machine, expected_response_code,file_name", DATA_STAT)
+def test_stat(machine, expected_response_code, file_name, headers):
+	url = f"{UTILITIES_URL}/stat"
+	params = {"targetPath": file_name}
+	headers.update({"X-Machine-Name": machine})
+	resp = requests.get(url, headers=headers, params=params, verify= (f"{SSL_PATH}{SSL_CRT}" if USE_SSL else False))
+	print(resp.content)
+	print(resp.headers)
+	assert resp.status_code == expected_response_code
+
+
 @skipif_not_uses_gateway
 @pytest.mark.parametrize("machine, expected_response_code", DATA)
 def test_file_type_error(machine, expected_response_code, headers):

src/utilities/utilities.py

@@ -91,6 +91,16 @@ def file_type():
     return common_fs_operation(request, "file")
 
 
+## stat: determines the status of a file
+## params:
+##  - path: Filesystem path (Str) *required
+##  - machinename: str *required
+
+@app.route("/stat", methods=["GET"])
+@check_auth_header
+def stat():
+    return common_fs_operation(request, "stat")
+
 ## chmod: Change Mode of path in Filesystem
 ## params:
 ##  - path: Filesystem path (Str) *required
@@ -221,7 +231,7 @@ def make_directory():
 @check_auth_header
 def view():
     try:
-        resp = common_fs_operation(request, "stat")
+        resp = common_fs_operation(request, "fsize")
         if resp[1] != 200:
             return resp
         out = json.loads(resp[0].data.decode())
@@ -295,7 +305,7 @@ def common_fs_operation(request, command):
     tn = 'targetPath'
     if request.method == 'GET':
         targetPath = request.args.get("targetPath", None)
-        if (targetPath == None) and (command in ['base64', 'stat']):
+        if (targetPath == None) and (command in ['base64', 'fsize', 'stat']):
             # TODO: review API
             tn = "sourcePath"
             targetPath = request.args.get("sourcePath", None)
@@ -367,8 +377,13 @@ def common_fs_operation(request, command):
         # -r is for recursivelly delete files into directories
         action = f"rm -r --interactive=never -- '{targetPath}'"
         success_code = 204
-    elif command == "stat":
+    elif command == "fsize":
         action = f"stat --dereference -c %s -- '{targetPath}'"
+    elif command == "stat":
+        deref = ""
+        if request.args.get("dereference", False):
+            deref = "--dereference"
+        action = f"stat {deref} -c '%a %i %d %h %u %g %s %X %Y %Z' -- '{targetPath}'"
     elif command == "symlink":
         linkPath = request.form.get("linkPath", None)
         v = validate_input(linkPath)
@@ -419,8 +434,14 @@ def common_fs_operation(request, command):
     if command == 'checksum':
         # return only hash, msg sintax:  hash filename
         output = retval["msg"].split()[0]
-    elif command in ['base64', 'chmod', 'chown', 'file', 'head', 'stat']:
+    elif command in ['base64', 'chmod', 'chown', 'file', 'head', 'fsize']:
         output = retval["msg"]
+    elif command == 'stat':
+        # follows: https://docs.python.org/3/library/os.html#os.stat_result
+        output = dict(zip(['mode', 'ino', 'dev', 'nlink', 'uid', 'gid', 'size', 'atime', 'mtime', 'ctime'], retval["msg"].split()))
+        # convert to integers
+        # output["mode"] = int(output["mode"], base=16)
+        output = {key: int(value) for key, value in output.items()}
     elif command == 'ls':
         description = "List of contents"
         output = ls_parse(request, retval)
@@ -464,7 +485,7 @@ def symlink():
 def download():
     try:
         # returns a tuple (json_msg, status_code [, header])
-        resp = common_fs_operation(request, "stat")
+        resp = common_fs_operation(request, "fsize")
         if resp[1] != 200:
             return resp
         out = json.loads(resp[0].data.decode())