Merge pull request #643 from MerginMaps/whitelisting_extensions_mimetypes

Support for whitelisting extensions/mimetype

Author: MarcelGeo

deployment/community/.env.template

@@ -109,6 +109,9 @@ LOCAL_PROJECTS=/data
 
 #BLACKLIST='.mergin/, .DS_Store, .directory'  # cast=Csv()
 
+# extra file extensions to permit beyond the default block-list, e.g. '.py, .sh'
+#UPLOAD_EXTENSIONS_WHITELIST=
+
 #FILE_EXPIRATION=48 * 3600  # for clean up of old files where diffs were applied, in seconds
 
 #LOCKFILE_EXPIRATION=300  # in seconds

server/mergin/sync/config.py

@@ -82,5 +82,9 @@ class Configuration(object):
     )
     # files that should be ignored during extension and MIME type checks
     UPLOAD_FILES_WHITELIST = config("UPLOAD_FILES_WHITELIST", default="", cast=Csv())
+    # extra extensions to permit beyond the default block-list
+    UPLOAD_EXTENSIONS_WHITELIST = config(
+        "UPLOAD_EXTENSIONS_WHITELIST", default="", cast=Csv()
+    )
     # max batch size for fetch projects in batch endpoint
     MAX_BATCH_SIZE = config("MAX_BATCH_SIZE", default=100, cast=int)

server/mergin/sync/utils.py

@@ -463,7 +463,10 @@ def check_skip_validation(file_path: str) -> bool:
     Some files are allowed even if they have forbidden extension or mime type.
     """
     file_name = os.path.basename(file_path)
-    return file_name in Configuration.UPLOAD_FILES_WHITELIST
+    if file_name in Configuration.UPLOAD_FILES_WHITELIST:
+        return True
+    ext = os.path.splitext(file_path)[1].lower()
+    return ext in {e.lower() for e in Configuration.UPLOAD_EXTENSIONS_WHITELIST}
 
 
 FORBIDDEN_MIME_TYPES = {

server/mergin/tests/test_utils.py

@@ -402,3 +402,31 @@ def test_mime_type_validation_skip():
 
         # Should be forbidden
         assert not is_supported_type("other.js")
+
+
+def test_allowed_extensions_override():
+    """Extensions in UPLOAD_EXTENSIONS_WHITELIST are accepted even though they are in FORBIDDEN_EXTENSIONS."""
+    with patch(
+        "mergin.sync.utils.Configuration.UPLOAD_EXTENSIONS_WHITELIST", [".py", ".sh"]
+    ):
+        # forbidden by default, now explicitly allowed
+        assert is_supported_extension("model.py")
+        assert is_supported_extension("scripts/deploy.sh")
+        # match is case-insensitive
+        assert is_supported_extension("MODEL.PY")
+        # extensions not in the override stay blocked
+        assert not is_supported_extension("malware.exe")
+        assert not is_supported_extension("app.js")
+
+
+def test_extension_whitelist_skips_mime_check():
+    """A whitelisted extension also bypasses the MIME check via check_skip_validation."""
+    with patch("mergin.sync.utils.get_mimetype", return_value="text/x-shellscript"):
+        # blocked when the extension is not whitelisted
+        with patch("mergin.sync.utils.Configuration.UPLOAD_EXTENSIONS_WHITELIST", []):
+            assert not is_supported_type("deploy.sh")
+        # allowed once the extension is whitelisted
+        with patch(
+            "mergin.sync.utils.Configuration.UPLOAD_EXTENSIONS_WHITELIST", [".sh"]
+        ):
+            assert is_supported_type("deploy.sh")