Merge branch 'develop' of https://git.gieszer.link/gcrkrause/mastodon-blocklist-deploy into develop

Reviewed-on: https://git.gieszer.link/gcrkrause/mastodon-blocklist-deploy/pulls/6

DEVELOPMENT.md

@@ -0,0 +1,17 @@
+# Development Guide
+
+## Docker
+
+In order to have a common development environment, its nice to use docker. Its quite easy. To build a new image, simply run 
+
+`docker build . -t mastodon_blocklist_deploy`
+
+Now you can execute any commands using
+
+`docker run --rm mastodon_blocklist_deploy --help`
+
+If you want to avoid building new containers for each change, simply mount your code into the container using
+
+`docker run --rm -v $(pwd):/app mastodon_blocklist_deploy`
+
+Please be aware that changes to the package itself require a rebuild anyways.

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+COPY pyproject.toml poetry.lock README.md /app/
+COPY mastodon_blocklist_deploy /app/mastodon_blocklist_deploy
+WORKDIR /app
+
+ENTRYPOINT ["mastodon_blocklist_deploy"]
+
+RUN pip install -e . 

README.md

@@ -14,6 +14,77 @@ and [remove](https://docs.joinmastodon.org/methods/admin/domain_blocks/#delete)
 and [add](https://docs.joinmastodon.org/methods/admin/domain_blocks/#create) newly added.
 
 Since we have several attributes for a domain blog, a simple `.txt` file might not be sufficient. We probably want to
-set the severity, reject_media, reject_reports and comments. This means we need a human readable, easily python-readable
+set the severity, reject_media, reject_reports and comments. This means we need a human-readable, easily python-readable
 and structured file format. Since Python 3.11 got native support for [toml](https://toml.io/) and it
 supports [Array of Tables](https://toml.io/en/v1.0.0#array-of-tables), I'd prefer to use this.
+
+
+# Basic usage
+
+
+##
+
+```
+$ mastodon_blocklist_deploy -h
+usage: mastodon_blocklist_deploy [-h] [-s SERVER] [-t TOKEN] [-i INPUT_FILE] [-r REMOTE_BLOCKLIST] [-o OUTPUT] [-v] [-n] {diff,deploy,export}
+
+Deploy blocklist updates to a mastodon server
+
+positional arguments:
+  {diff,deploy,export}  Either use 'diff' to check the difference between local blockĺist and the blocklist on the server, 'deploy' to apply the current local blocklist or 'export' to export the remote blocklist into a local file.
+
+options:
+  -h, --help            show this help message and exit
+  -s SERVER, --server SERVER
+                        The address of the server where you want to deploy (e.g. mastodon.social)
+  -t TOKEN, --token TOKEN
+                        Authorization token
+  -i INPUT_FILE, --input-file INPUT_FILE
+                        The blocklist to use
+  -r REMOTE_BLOCKLIST, --remote-blocklist REMOTE_BLOCKLIST
+                        The remote blocklist as json for debugging reasons
+  -o OUTPUT, --output OUTPUT
+                        Filename where to export the blocklist
+  -v, --verbose
+  -n, --no-delete       Do not delete existing blocks
+```
+
+## Obtain a server token
+
+1. Be an admin on the server.
+2. Add an application in the Mastodon Web Client (https://yourdomain.org/settings/applications/new. Make sure to select the permissions `admin:read` and `admin:write`.
+3. Copy the Token (last value in the table) ![](assets/obtain_token.png)
+
+# Typical workflow
+
+1. **Export the current blocklist from the server**
+
+```
+mastodon_blocklist_deploy export -s yourserver -t yourtoken -o blocklist.toml
+```
+
+2. **Manually add something to the blocklist**
+
+```toml
+[[instances]]
+name = "instance-to-block.com"
+domain = "instance-to-block.com"
+severity = "suspend"
+reject_media = true
+reject_reports = true
+public_comment = "X, Y and Z"
+private_comment = "We discussed this after X and Y and now that Z happend we decided to block"
+```
+
+3. **Check the difference between the local and remote blocklist**
+
+```
+mastodon_blocklist_deploy diff -s yourserver -t yourtoken -i blocklist.toml
+```
+
+
+4. **Apply the local blocklist to the server**
+
+```
+mastodon_blocklist_deploy apply -s yourserver -t yourtoken -i blocklist.toml
+```

mastodon_blocklist_deploy/cli.py

@@ -3,13 +3,13 @@ import argparse
 import json
 import logging
 import requests
-
+import os
 import toml
 
-from models import Instance
+from mastodon_blocklist_deploy.models import Instance
 
 
-def load_local_blocklist(filename: str) -> [Instance]:
+def load_blocklist_file(filename: str) -> [Instance]:
     with open(filename, "r") as f:
         data = toml.load(f)
     instances = []
@@ -19,31 +19,14 @@ def load_local_blocklist(filename: str) -> [Instance]:
     return instances
 
 
-def export_blocklist_toml(blocklist: [Instance], filname: str):
-    toml_str = ""
-    for instance in blocklist:
-        toml_str += f'''
-[[instances]]
-name = "{instance.domain}"
-domain = "{instance.domain}"
-severity = "{instance.severity}"
-reject_media = {str(instance.reject_media).lower()}
-reject_reports = {str(instance.reject_reports).lower()}
-public_comment = "{instance.public_comment}"
-private_comment = "{instance.private_comment}"
-        '''
-    with open(filname, "w") as f:
-        f.write(toml_str)
-
-
-def blocklist_json_to_instances(blocklist_json: str):
+def blocklist_json_to_instances(blocklist_json: str) -> [Instance]:
     instances = []
     for i in blocklist_json:
         instances.append(Instance(i))
     return instances
 
 
-def load_remote_blocklist(server: str, token: str):
+def load_blocklist_from_instance(server: str, token: str) -> [Instance]:
     headers = {
         f'Authorization': f'Bearer {token}',
     }
@@ -59,8 +42,8 @@ def load_remote_blocklist(server: str, token: str):
 def cli():
     parser = argparse.ArgumentParser(description='Deploy blocklist updates to a mastodon server')
     parser.add_argument('action', choices=['diff', 'deploy', 'export'],
-                        help="Either use 'diff' to check the difference between current blocks and future blocks, "
-                             "'deploy' to apply the current local blocklist or 'export' to export the remote "
+                        help="Either use 'diff' to check the difference between local blockĺist and the blocklist on "
+                             "the server, 'deploy' to apply the current local blocklist or 'export' to export the remote "
                              "blocklist into a local file.")
     parser.add_argument('-s', '--server', help="The address of the server where you want to deploy (e.g. "
                                                "mastodon.social)")
@@ -68,36 +51,51 @@ def cli():
     parser.add_argument('-i', '--input-file', help="The blocklist to use")
     parser.add_argument('-r', '--remote-blocklist', help="The remote blocklist as json for debugging reasons")
     parser.add_argument('-o', '--output', help="Filename where to export the blocklist")
-    parser.add_argument('-v', '--verbose',
-                        action='store_true')
+    parser.add_argument('-v', '--verbose', action='store_true')
+    parser.add_argument('-n', '--no-delete', action='store_true', help="Do not delete existing blocks")
     args = parser.parse_args()
     if args.verbose:
         logging.basicConfig(level=logging.DEBUG)
     else:
         logging.basicConfig(level=logging.WARN)
 
+    if args.token:
+        token = args.token
+    else:
+        token = os.getenv('MBD_TOKEN')
+
+
+
     """if there is a remote blocklist provided load this instead of fetching it from a server (for debugging reasons)"""
     if args.remote_blocklist:
         with open(args.remote_blocklist) as f:
             remote_blocklist = blocklist_json_to_instances(json.load(f))
     else:
-        remote_blocklist = load_remote_blocklist(server=args.server, token=args.token)
+        remote_blocklist = load_blocklist_from_instance(server=args.server, token=token)
 
     """Load local blocklist only when needed"""
     if args.action in ["diff", "deploy"]:
         if args.input_file:
             blocklist_filename = args.input_file
         else:
-            blocklist_filename = "blocklist.toml"
-        local_blocklist = load_local_blocklist(blocklist_filename)
+            blocklist_filename = "../blocklist.toml"
+        try:
+            local_blocklist = load_blocklist_file(blocklist_filename)
+        except FileNotFoundError:
+            print("Local blocklist file was not found. Make sure to specify it's location via -i")
+            exit()
 
     if args.action == "diff":
         Instance.show_diffs(local_blocklist, remote_blocklist)
     elif args.action == "deploy":
         diffs = Instance.list_diffs(local_blocklist, remote_blocklist)
-        Instance.apply_blocks_from_diff(diffs, args.server, args.token)
+        Instance.apply_blocks_from_diff(diffs, args.server, token, args.no_delete)
     elif args.action == "export":
-        export_blocklist_toml(remote_blocklist, args.output)
+        if not args.output:
+            print(toml.dumps({"instances": [b.__dict__ for b in remote_blocklist]}))
+        else:
+            with open(args.output, "w") as f:
+                toml.dump({"instances": [b.__dict__ for b in remote_blocklist]}, f)
 
 
 if __name__ == "__main__":

mastodon_blocklist_deploy/models.py

@@ -25,7 +25,7 @@ class Instance:
         return self.domain == other.domain and self.severity == other.severity and self.reject_media == other.reject_media and self.reject_reports == other.reject_reports and self.obfuscate == other.obfuscate
 
     def status_str(self):
-        return f"{self.severity}, Reject reports: {self.reject_reports}, Reject media: {self.reject_media}, Obfuscate: {self.obfuscate}"
+        return f"{self.severity}\nReject reports: {self.reject_reports}\nReject media: {self.reject_media}\nObfuscate: {self.obfuscate}"
 
     def parse_remote_block(self, instance_dict):
         self.domain = instance_dict["domain"]
@@ -57,24 +57,28 @@ class Instance:
             pass
 
     def apply(self, server, token, block_id=None):
+        """Applies instance block on the remote server"""
         headers = {
             f'Authorization': f'Bearer {token}',
         }
         data = {"domain": self.domain,
                 "severity": self.severity,
-                "reject_media": self.reject_media,
-                "reject_reports": self.reject_reports,
-                "private_comment": self.private_comment,
+                "reject_media": str(self.reject_media).lower(),
+                "reject_reports": str(self.reject_reports).lower(),
+                "private_comment": str(self.private_comment).lower(),
                 "public_comment": self.public_comment,
-                "obfuscate": self.obfuscate}
+                "obfuscate": str(self.obfuscate).lower()}
         """If no id is given add a new block, else update the existing block"""
         if block_id is None:
             response = requests.post(f'https://{server}/api/v1/admin/domain_blocks', data=data, headers=headers)
         else:
-            response = requests.put(f'https://{server}/api/v1/admin/domain_blocks/{block_id}', data=data, headers=headers)
+            response = requests.put(f'https://{server}/api/v1/admin/domain_blocks/{block_id}', data=data,
+                                    headers=headers)
         if response.status_code != 200:
             raise ConnectionError(f"Could not apply block ({response.status_code}: {response.reason})")
+
     def delete(self, server: str, token: str):
+        """Deletes the instance from the blocklist on the remote server"""
         headers = {
             f'Authorization': f'Bearer {token}',
         }
@@ -82,9 +86,9 @@ class Instance:
         if response.status_code != 200:
             raise ConnectionError(f"Could not apply block ({response.status_code}: {response.reason})")
 
-
     @staticmethod
     def list_diffs(local_blocklist, remote_blocklist):
+        """Compares the local and remote blocklist and returns a list of differences"""
         diffs = []
         for local_instance in local_blocklist:
             instance_found = False
@@ -107,30 +111,33 @@ class Instance:
         return diffs
 
     @staticmethod
-    def apply_blocks_from_diff(diffs, server, token):
+    def apply_blocks_from_diff(diffs, server, token, no_delete: bool):
+        """Uses a diff (list of difference in local an remote instance) to apply instance blocks"""
         for diff in diffs:
             if diff["local"] is None:
-                """Delete the block on the remote server"""
-                diff['remote'].delete(server, token)
-                logging.info(f"Deleted {diff['remote'].domain} from blocklist")
+                if not no_delete:
+                    """Delete the block on the remote server"""
+                    diff['remote'].delete(server, token)
+                    logging.info(f"Deleted {diff['remote'].domain} from blocklist")
             elif diff["remote"] is None:
                 """Add the block on the remote server"""
                 diff["local"].apply(server, token)
-                logging.info(f"Added {diff['remote'].domain} to blocklist")
+                logging.info(f"Added {diff['local'].domain} to blocklist")
             else:
                 """Update the block on the remote server"""
                 diff["local"].apply(server, token, block_id=diff["remote"].id)
-                logging.info(f"Updated {diff['remote'].domain} in blocklist")
+                logging.info(f"Updated {diff['local'].domain} in blocklist")
 
     @staticmethod
     def show_diffs(local_blocklist, remote_blocklist):
+        """Shows a table in the CLI comparing the local and remote blocklist"""
         from rich.table import Table
         from rich.console import Console
         table = Table(title="Differences", expand=True, show_lines=True)
 
         table.add_column("Domain", style="cyan")
-        table.add_column("Current remote status", style="magenta")
         table.add_column("Local status", style="green")
+        table.add_column("Current remote status", style="magenta")
         diffs = Instance.list_diffs(local_blocklist, remote_blocklist)
         for diff in diffs:
             if diff["local"] is None:

poetry.lock

@@ -0,0 +1,145 @@
+# This file is automatically @generated by Poetry and should not be changed by hand.
+
+[[package]]
+name = "certifi"
+version = "2022.12.7"
+description = "Python package for providing Mozilla's CA Bundle."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"},
+    {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"},
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "2.1.1"
+description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+category = "main"
+optional = false
+python-versions = ">=3.6.0"
+files = [
+    {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"},
+    {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"},
+]
+
+[package.extras]
+unicode-backport = ["unicodedata2"]
+
+[[package]]
+name = "commonmark"
+version = "0.9.1"
+description = "Python parser for the CommonMark Markdown spec"
+category = "main"
+optional = false
+python-versions = "*"
+files = [
+    {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"},
+    {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"},
+]
+
+[package.extras]
+test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"]
+
+[[package]]
+name = "idna"
+version = "3.4"
+description = "Internationalized Domain Names in Applications (IDNA)"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"},
+    {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"},
+]
+
+[[package]]
+name = "pygments"
+version = "2.14.0"
+description = "Pygments is a syntax highlighting package written in Python."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"},
+    {file = "Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"},
+]
+
+[package.extras]
+plugins = ["importlib-metadata"]
+
+[[package]]
+name = "requests"
+version = "2.28.1"
+description = "Python HTTP for Humans."
+category = "main"
+optional = false
+python-versions = ">=3.7, <4"
+files = [
+    {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"},
+    {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"},
+]
+
+[package.dependencies]
+certifi = ">=2017.4.17"
+charset-normalizer = ">=2,<3"
+idna = ">=2.5,<4"
+urllib3 = ">=1.21.1,<1.27"
+
+[package.extras]
+socks = ["PySocks (>=1.5.6,!=1.5.7)"]
+use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
+
+[[package]]
+name = "rich"
+version = "13.0.1"
+description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+category = "main"
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "rich-13.0.1-py3-none-any.whl", hash = "sha256:41fe1d05f433b0f4724cda8345219213d2bfa472ef56b2f64f415b5b94d51b04"},
+    {file = "rich-13.0.1.tar.gz", hash = "sha256:25f83363f636995627a99f6e4abc52ed0970ebbd544960cc63cbb43aaac3d6f0"},
+]
+
+[package.dependencies]
+commonmark = ">=0.9.0,<0.10.0"
+pygments = ">=2.6.0,<3.0.0"
+
+[package.extras]
+jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"]
+
+[[package]]
+name = "toml"
+version = "0.10.2"
+description = "Python Library for Tom's Obvious, Minimal Language"
+category = "main"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+files = [
+    {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
+    {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
+]
+
+[[package]]
+name = "urllib3"
+version = "1.26.14"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+files = [
+    {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"},
+    {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"},
+]
+
+[package.extras]
+brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
+secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "^3.10"
+content-hash = "f6e631371be67516f200e86805f1fab33dcd481a779c93f97a510f7348d0e2aa"

pyproject.toml

@@ -5,6 +5,9 @@ description = "A small tool to deploy blocklist updates to a mastodon server usi
 authors = ["Georg Krause <mail@georg-krause.net>", "Julian-Samuel Gebühr <julian-samuel@gebuehr.net>"]
 readme = "README.md"
 packages = [{include = "mastodon_blocklist_deploy"}]
+license = "MIT"
+keywords = ["mastodon", "blocklist", "fediverse"]
+
 
 [tool.poetry.dependencies]
 python = "^3.10"
@@ -12,6 +15,9 @@ requests = "^2.28.1"
 rich = "^13.0.1"
 toml = "^0.10.2"
 
+[tool.poetry.scripts]
+mastodon_blocklist_deploy = 'mastodon_blocklist_deploy.cli:cli'
+
 
 [build-system]
 requires = ["poetry-core"]