Cleanup and add a unique request ID

utils/blacklist.py

@@ -0,0 +1,38 @@
+import logging
+from typing import Any
+from typing import Dict
+from urllib.parse import urlparse
+
+import config
+
+logger = logging.getLogger(__name__)
+
+
+def is_url_blacklisted(url: str) -> bool:
+    try:
+        return urlparse(url).netloc in config.BLACKLIST
+    except Exception:
+        logger.exception(f"failed to blacklist for {url}")
+        return False
+
+
+def is_blacklisted(data: Dict[str, Any]) -> bool:
+    """Returns True if the activity is coming/or referencing a blacklisted host."""
+    if (
+        "id" in data
+        and is_url_blacklisted(data["id"])
+        or (
+            "object" in data
+            and isinstance(data["object"], dict)
+            and "id" in data["object"]
+            and is_url_blacklisted(data["object"]["id"])
+        )
+        or (
+            "object" in data
+            and isinstance(data["object"], str)
+            and is_url_blacklisted(data["object"])
+        )
+    ):
+        return True
+
+    return False

utils/media.py

@@ -1,6 +1,7 @@
 import base64
 import mimetypes
 from enum import Enum
+from enum import unique
 from gzip import GzipFile
 from io import BytesIO
 from typing import Any
@@ -13,7 +14,7 @@ from little_boxes import activitypub as ap
 from PIL import Image
 
 
-def load(url, user_agent):
+def load(url: str, user_agent: str) -> Image:
     """Initializes a `PIL.Image` from the URL."""
     with requests.get(url, stream=True, headers={"User-Agent": user_agent}) as resp:
         resp.raise_for_status()
@@ -22,7 +23,7 @@ def load(url, user_agent):
         return Image.open(BytesIO(resp.raw.read()))
 
 
-def to_data_uri(img):
+def to_data_uri(img: Image) -> str:
     out = BytesIO()
     img.save(out, format=img.format)
     out.seek(0)
@@ -30,6 +31,7 @@ def to_data_uri(img):
     return f"data:{img.get_format_mimetype()};base64,{data}"
 
 
+@unique
 class Kind(Enum):
     ATTACHMENT = "attachment"
     ACTOR_ICON = "actor_icon"

utils/opengraph.py

@@ -1,4 +1,7 @@
 import logging
+from typing import Any
+from typing import Dict
+from typing import Set
 from urllib.parse import urlparse
 
 import opengraph
@@ -14,7 +17,7 @@ from .lookup import lookup
 logger = logging.getLogger(__name__)
 
 
-def links_from_note(note):
+def links_from_note(note: Dict[str, Any]) -> Set[str]:
     note_host = urlparse(ap._get_id(note["id"]) or "").netloc
 
     links = set()

utils/template_filters.py

@@ -170,7 +170,6 @@ def url_or_id(d):
 
 @filters.app_template_filter()
 def get_url(u):
-    print(f"GET_URL({u!r})")
     if isinstance(u, list):
         for l in u:
             if l.get("mimeType") == "text/html":
@@ -191,7 +190,6 @@ def get_actor(url):
         url = url[0]
     if isinstance(url, dict):
         url = url.get("id")
-    print(f"GET_ACTOR {url}")
     try:
         return ap.get_backend().fetch_iri(url)
     except (ActivityNotFoundError, ActivityGoneError):
@@ -203,8 +201,6 @@ def get_actor(url):
 @filters.app_template_filter()
 def get_answer_count(choice, obj, meta):
     count_from_meta = meta.get("question_answers", {}).get(_answer_key(choice), 0)
-    print(count_from_meta)
-    print(choice, obj, meta)
     if count_from_meta:
         return count_from_meta
     for option in obj.get("oneOf", obj.get("anyOf", [])):
@@ -219,7 +215,6 @@ def get_total_answers_count(obj, meta):
         return cached
     cnt = 0
     for choice in obj.get("anyOf", obj.get("oneOf", [])):
-        print(choice)
         cnt += choice.get("replies", {}).get("totalItems", 0)
     return cnt