Add merge functionality

README.md

@@ -9,11 +9,11 @@ the mastodon server, this is supposed to be automated with Drone CI.
 
 In order to compare the list entries, we can read the whole blocklist
 using [the get endpoint](https://docs.joinmastodon.org/methods/admin/domain_blocks/#get). At the same time we read the
-whole file in the repository, make a comparision
+whole file in the repository, make a comparison
 and [remove](https://docs.joinmastodon.org/methods/admin/domain_blocks/#delete) unblocked domains from the blocklist
 and [add](https://docs.joinmastodon.org/methods/admin/domain_blocks/#create) newly added.
 
-Since we have several attributes for a domain blog, a simple `.txt` file might not be sufficient. We probably want to
+Since we have several attributes for a domain block, a simple `.txt` file might not be sufficient. We probably want to
 set the severity, reject_media, reject_reports and comments. This means we need a human-readable, easily python-readable
 and structured file format. Since Python 3.11 got native support for [toml](https://toml.io/) and it
 supports [Array of Tables](https://toml.io/en/v1.0.0#array-of-tables), I'd prefer to use this.
@@ -49,7 +49,7 @@ options:
                         Filename where to export the blocklist
   -v, --verbose
   -n, --no-delete       Do not delete existing blocks
-  --format FORMAT       Export format: toml|markdown|csv
+  --format FORMAT       Export format: toml|markdown|csv|json
   --private             When the flag is set, private comment will also be exported.
 ```
 
@@ -91,4 +91,4 @@ mastodon_blocklist_deploy diff -s yourserver -t yourtoken -i blocklist.toml
 
 ```
 mastodon_blocklist_deploy apply -s yourserver -t yourtoken -i blocklist.toml
-```
+```

mastodon_blocklist_deploy/cli.py

@@ -7,7 +7,8 @@ import os
 import toml
 
 from mastodon_blocklist_deploy.models import Instance
-from mastodon_blocklist_deploy.helpers import blocklist_to_markdown, blocklist_to_toml, blocklist_to_csv
+from mastodon_blocklist_deploy.helpers import blocklist_to_markdown, blocklist_to_toml, blocklist_to_csv, \
+    blocklist_to_json
 
 
 def load_blocklist_file(filename: str) -> [Instance]:
@@ -52,6 +53,8 @@ def exporter(blocklist, output=None, format: str = "toml", private: bool = False
         exported_text = blocklist_to_csv(blocklist, private)
     if format == "markdown":
         exported_text = blocklist_to_markdown(blocklist, private)
+    if format == "json":
+        exported_text = blocklist_to_json(blocklist, private)
 
     # Output the text
     if output is not None:
@@ -61,13 +64,37 @@ def exporter(blocklist, output=None, format: str = "toml", private: bool = False
         print(exported_text)
 
 
+def merge(input_file, merge_target, format: str = "toml", private: bool = False, overwrite=False):
+    input_blocklist = load_blocklist_file(input_file)
+    merge_target_blocklist = load_blocklist_file(merge_target)
+    for input_instance in input_blocklist:
+        # If the block is already there with the same parameters we do nothing
+        if input_instance in merge_target_blocklist:
+            continue
+        # Check if there is a domain in the merge target where the input domain is similar
+        try:
+            merge_target_instance = [merge_target_instance for merge_target_instance in merge_target if input_instance.domain == merge_target_instance.domain][0]
+            if not overwrite:
+                key_input = ""
+                while key_input not in ("i", "O"):
+                    print(f"Different settings for {input_instance.domain} detected.")
+                    print(f"In the input blocklist the setting is\n{input_instance} whereas it's {merge_target_instance} in the merge target")
+                    key_input = input("Keep input (i) or original (o) [i/O]")
+            elif key_input == "i":
+                merge_target_blocklist.append(merge_target_instance)
+            else:
+                merge_target_blocklist.append(input_instance)
+        except KeyError:
+            pass
+
 
 def cli():
     parser = argparse.ArgumentParser(description='Deploy blocklist updates to a mastodon server')
-    parser.add_argument('action', choices=['diff', 'deploy', 'export'],
+    parser.add_argument('action', choices=['diff', 'deploy', 'export', 'merge'],
                         help="Either use 'diff' to check the difference between local blockĺist and the blocklist on "
                              "the server, 'deploy' to apply the current local blocklist or 'export' to export the remote "
-                             "blocklist into a local file.")
+                             "blocklist into a local file. merge can be used to merge a blocklist (given by -i) into "
+                             "another (-o)")
     parser.add_argument('-s', '--server', help="The address of the server where you want to deploy (e.g. "
                                                "mastodon.social)")
     parser.add_argument('-t', '--token', help="Authorization token")
@@ -76,7 +103,7 @@ def cli():
     parser.add_argument('-o', '--output', help="Filename where to export the blocklist")
     parser.add_argument('-v', '--verbose', action='store_true')
     parser.add_argument('-n', '--no-delete', action='store_true', help="Do not delete existing blocks")
-    parser.add_argument('--format', help="Export format: toml|markdown|csv")
+    parser.add_argument('--format', help="Export format: toml|markdown|csv|json")
     parser.add_argument('--private', action='store_true', help="When the flag is set, private comment will also be "
                                                                "exported.")
     args = parser.parse_args()
@@ -90,8 +117,6 @@ def cli():
     else:
         token = os.getenv('MBD_TOKEN')
 
-
-
     """if there is a remote blocklist provided load this instead of fetching it from a server (for debugging reasons)"""
     if args.remote_blocklist:
         with open(args.remote_blocklist) as f:
@@ -100,7 +125,7 @@ def cli():
         remote_blocklist = load_blocklist_from_instance(server=args.server, token=token)
 
     """Load local blocklist only when needed"""
-    if args.action in ["diff", "deploy"]:
+    if args.action in ["diff", "deploy", "merge"]:
         if args.input_file:
             blocklist_filename = args.input_file
         else:
@@ -118,6 +143,9 @@ def cli():
         Instance.apply_blocks_from_diff(diffs, args.server, token, args.no_delete)
     elif args.action == "export":
         exporter(remote_blocklist, args.output, args.format, args.private)
+    elif args.action == "merge":
+        merge(args.input_file, args.output, args.format, args.private)
+
 
 if __name__ == "__main__":
     cli()

mastodon_blocklist_deploy/helpers.py

@@ -2,6 +2,7 @@ from mastodon_blocklist_deploy.models import Instance
 import toml
 import io
 import csv
+import json
 
 def blocklist_to_markdown(blocklist: [Instance], private: bool = False):
     if private:
@@ -29,3 +30,7 @@ def blocklist_to_csv(blocklist: [Instance], private: bool = False):
     w.writeheader()
     w.writerows(blocklist_as_dict)
     return csv_string.getvalue()
+
+def blocklist_to_json(blocklist: [Instance], private: bool = False):
+    json_string = json.dumps([b.as_dict(private) for b in blocklist])
+    return json_string

mastodon_blocklist_deploy/models.py

@@ -11,12 +11,7 @@ class Instance:
         self.reject_reports = False
         self.id = None
 
-        """Remote blocks and local blocks are parsed differently"""
-        try:
-            instance_dict["id"]
-            self.parse_remote_block(instance_dict)
-        except KeyError:
-            self.parse_local_block(instance_dict)
+        self.parse_block(instance_dict)
 
     def __str__(self):
         return f"{self.domain}: {self.severity}"
@@ -36,37 +31,20 @@ class Instance:
             exportable[key] = getattr(self, key)
         return exportable
 
-    def parse_remote_block(self, instance_dict):
-        self.domain = instance_dict["domain"]
-        self.id = instance_dict["id"]
-        self.severity = instance_dict["severity"]
-        self.public_comment = instance_dict["public_comment"]
-        self.private_comment = instance_dict["private_comment"]
-        self.obfuscate = instance_dict["obfuscate"]
-        self.reject_media = instance_dict["reject_media"]
-        self.reject_reports = instance_dict["reject_reports"]
-
-    def parse_local_block(self, instance_dict):
-        try:
-            self.name = instance_dict["name"]
-        except KeyError:
-            pass
-        self.domain = instance_dict["domain"]
-        self.severity = instance_dict["severity"]
-        self.public_comment = instance_dict["public_comment"]
-        self.private_comment = instance_dict["private_comment"]
-        try:
-            self.obfuscate = instance_dict["obfuscate"]
-        except KeyError:
-            pass
-        try:
-            self.reject_media = instance_dict["reject_media"]
-        except KeyError:
-            pass
-        try:
-            self.reject_reports = instance_dict["reject_reports"]
-        except KeyError:
-            pass
+    def parse_block(self, instance_dict):
+        # this specifies possible properties and default values if not found on the remote source. If a default is None
+        # the value is required and the parse will fail
+        properties_and_defaults = [("domain", None), ("severity", "suspend"), ("public_comment", ""),
+                                   ("private_comment", ""), ("obfuscate", False), ("reject_media", True),
+                                   ("reject_reports", True)]
+        for key, default in properties_and_defaults:
+            try:
+                setattr(self, key, instance_dict[key])
+            except KeyError:
+                if default is not None:
+                    setattr(self, key, default)
+                else:
+                    raise KeyError(f"The key {key} was not in the instance_dict response.")
 
     def apply(self, server, token, block_id=None):
         """Applies instance block on the remote server"""