From ac0749797fc0d4c92451ff7c7fab80ebb6d16298 Mon Sep 17 00:00:00 2001 From: Salil <32305505+Deadpool2000@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:03:30 +0530 Subject: [PATCH 01/10] Create robots.txt - robots.txt file added in src/fellchensammlung/static/ --- src/fellchensammlung/static/robots.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/fellchensammlung/static/robots.txt diff --git a/src/fellchensammlung/static/robots.txt b/src/fellchensammlung/static/robots.txt new file mode 100644 index 0000000..a9ac2a6 --- /dev/null +++ b/src/fellchensammlung/static/robots.txt @@ -0,0 +1,7 @@ +User-agent: * +Disallow: /admin/ + +User-agent: OpenAI +Disallow: / + +Sitemap: https://notfellchen.org/sitemap.xml From d7269106db9d07a09191dc37c674b6bd6582c87e Mon Sep 17 00:00:00 2001 From: Salil <32305505+Deadpool2000@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:22:28 +0530 Subject: [PATCH 02/10] Added - animal_shelter Get Data Import all German animal shelters --- src/fellchensammlung/animal_shelter.py | 57 ++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 src/fellchensammlung/animal_shelter.py diff --git a/src/fellchensammlung/animal_shelter.py b/src/fellchensammlung/animal_shelter.py new file mode 100644 index 0000000..9fe1060 --- /dev/null +++ b/src/fellchensammlung/animal_shelter.py @@ -0,0 +1,57 @@ +import json +import requests + +OSM_DATA_FILE = "osm_data.geojson" +ENDPOINT = "https://test.notfellchen.org/api/organizations/" +HEADERS = { + "Authorization": "API_KEY", + "Content-Type": "application/json" +} + +def load_osm_data(file_path): + #Load OSM data from a GeoJSON file. + with open(file_path, "r", encoding="utf-8") as file: + data = json.load(file) + return data + +def transform_osm_data(feature): + #Transform a single OSM feature into the API payload format + prop = feature.get("prop", {}) + geometry = feature.get("geometry", {}) + + return { + "name": prop.get("name", "Unnamed Shelter"), + "phone": prop.get("phone"), + "website": prop.get("website"), + "opening_hours": prop.get("opening_hours"), + "email": prop.get("email"), + "location": { + "type": geometry.get("type", "Point"), + "coordinates": geometry.get("coordinates", []) + }, + "external_object_id": prop.get("@id"), + "external_source_id": "OSM" + } + +def send_to_api(data): + #Send transformed data to the Notfellchen API. + response = requests.post(ENDPOINT, headers=HEADERS, json=data) + if response.status_code == 201: + print(f"Success: Shelter '{data['name']}' uploaded.") + elif response.status_code == 400: + print(f"Error: Shelter '{data['name']}' already exists or invalid data.") + else: + print(f"Unexpected Error: {response.status_code} - {response.text}") + +def main(): + # Step 1: Load OSM data + osm_data = load_osm_data(OSM_DATA_FILE) + + + # Step 2: Process each shelter and send it to the API + for feature in osm_data.get("features", []): + shelter_data = transform_osm_data(feature) + send_to_api(shelter_data) + +if __name__ == "__main__": + main() From 461abd2e46d5f961ce8dd2b6022e3e4ae264cac7 Mon Sep 17 00:00:00 2001 From: moanos Date: Sun, 26 Jan 2025 17:04:17 +0100 Subject: [PATCH 03/10] fix: don't try to save owner --- src/fellchensammlung/api/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fellchensammlung/api/views.py b/src/fellchensammlung/api/views.py index f2b4b25..bec2a77 100644 --- a/src/fellchensammlung/api/views.py +++ b/src/fellchensammlung/api/views.py @@ -16,6 +16,7 @@ from .serializers import ( from fellchensammlung.models import Animal, RescueOrganization, AdoptionNotice, Species, Image from drf_spectacular.utils import extend_schema + class AdoptionNoticeApiView(APIView): permission_classes = [IsAuthenticated] @@ -84,7 +85,6 @@ class AdoptionNoticeApiView(APIView): ) - class AnimalApiView(APIView): permission_classes = [IsAuthenticated] @@ -118,6 +118,7 @@ class AnimalApiView(APIView): ) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + class RescueOrganizationApiView(APIView): permission_classes = [IsAuthenticated] @@ -159,13 +160,14 @@ class RescueOrganizationApiView(APIView): """ serializer = RescueOrgSerializer(data=request.data, context={"request": request}) if serializer.is_valid(): - rescue_org = serializer.save(owner=request.user) + rescue_org = serializer.save() return Response( {"message": "Rescue organization created/updated successfully!", "id": rescue_org.id}, status=status.HTTP_201_CREATED, ) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + class AddImageApiView(APIView): permission_classes = [IsAuthenticated] From 70e2af61722cf6bc7b50e218178fd230b11f58fb Mon Sep 17 00:00:00 2001 From: moanos Date: Sun, 26 Jan 2025 17:04:56 +0100 Subject: [PATCH 04/10] fix: fix key --- src/fellchensammlung/animal_shelter.py | 114 ++++++++++++------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/src/fellchensammlung/animal_shelter.py b/src/fellchensammlung/animal_shelter.py index 9fe1060..f727dbb 100644 --- a/src/fellchensammlung/animal_shelter.py +++ b/src/fellchensammlung/animal_shelter.py @@ -1,57 +1,57 @@ -import json -import requests - -OSM_DATA_FILE = "osm_data.geojson" -ENDPOINT = "https://test.notfellchen.org/api/organizations/" -HEADERS = { - "Authorization": "API_KEY", - "Content-Type": "application/json" -} - -def load_osm_data(file_path): - #Load OSM data from a GeoJSON file. - with open(file_path, "r", encoding="utf-8") as file: - data = json.load(file) - return data - -def transform_osm_data(feature): - #Transform a single OSM feature into the API payload format - prop = feature.get("prop", {}) - geometry = feature.get("geometry", {}) - - return { - "name": prop.get("name", "Unnamed Shelter"), - "phone": prop.get("phone"), - "website": prop.get("website"), - "opening_hours": prop.get("opening_hours"), - "email": prop.get("email"), - "location": { - "type": geometry.get("type", "Point"), - "coordinates": geometry.get("coordinates", []) - }, - "external_object_id": prop.get("@id"), - "external_source_id": "OSM" - } - -def send_to_api(data): - #Send transformed data to the Notfellchen API. - response = requests.post(ENDPOINT, headers=HEADERS, json=data) - if response.status_code == 201: - print(f"Success: Shelter '{data['name']}' uploaded.") - elif response.status_code == 400: - print(f"Error: Shelter '{data['name']}' already exists or invalid data.") - else: - print(f"Unexpected Error: {response.status_code} - {response.text}") - -def main(): - # Step 1: Load OSM data - osm_data = load_osm_data(OSM_DATA_FILE) - - - # Step 2: Process each shelter and send it to the API - for feature in osm_data.get("features", []): - shelter_data = transform_osm_data(feature) - send_to_api(shelter_data) - -if __name__ == "__main__": - main() +import json +import requests + +OSM_DATA_FILE = "osm_data.geojson" +ENDPOINT = "https://test.notfellchen.org/api/organizations/" +HEADERS = { + "Authorization": "API_KEY", + "Content-Type": "application/json" +} + +def load_osm_data(file_path): + #Load OSM data from a GeoJSON file. + with open(file_path, "r", encoding="utf-8") as file: + data = json.load(file) + return data + +def transform_osm_data(feature): + #Transform a single OSM feature into the API payload format + prop = feature.get("properties", {}) + geometry = feature.get("geometry", {}) + + return { + "name": prop.get("name", "Unnamed Shelter"), + "phone": prop.get("phone"), + "website": prop.get("website"), + "opening_hours": prop.get("opening_hours"), + "email": prop.get("email"), + "location": { + "type": geometry.get("type", "Point"), + "coordinates": geometry.get("coordinates", []) + }, + "external_object_id": prop.get("@id"), + "external_source_id": "OSM" + } + +def send_to_api(data): + #Send transformed data to the Notfellchen API. + response = requests.post(ENDPOINT, headers=HEADERS, json=data) + if response.status_code == 201: + print(f"Success: Shelter '{data['name']}' uploaded.") + elif response.status_code == 400: + print(f"Error: Shelter '{data['name']}' already exists or invalid data.") + else: + print(f"Unexpected Error: {response.status_code} - {response.text}") + +def main(): + # Step 1: Load OSM data + osm_data = load_osm_data(OSM_DATA_FILE) + + + # Step 2: Process each shelter and send it to the API + for feature in osm_data.get("features", []): + shelter_data = transform_osm_data(feature) + send_to_api(shelter_data) + +if __name__ == "__main__": + main() From 8858cff9cfd61d5db4706ced07642b15c08e8ff7 Mon Sep 17 00:00:00 2001 From: moanos Date: Sun, 26 Jan 2025 17:05:51 +0100 Subject: [PATCH 05/10] fix: Construct necessary location string I'd be better to directly create a location here but I for now want to make as little modifications as possible --- src/fellchensammlung/animal_shelter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/fellchensammlung/animal_shelter.py b/src/fellchensammlung/animal_shelter.py index f727dbb..74ba86c 100644 --- a/src/fellchensammlung/animal_shelter.py +++ b/src/fellchensammlung/animal_shelter.py @@ -25,10 +25,7 @@ def transform_osm_data(feature): "website": prop.get("website"), "opening_hours": prop.get("opening_hours"), "email": prop.get("email"), - "location": { - "type": geometry.get("type", "Point"), - "coordinates": geometry.get("coordinates", []) - }, + "location_string": f'{prop.get("addr:street", "")} {prop.get("addr:housenumber", "")} {prop.get("addr:postcode", "")} {prop.get("addr:city", "")}', "external_object_id": prop.get("@id"), "external_source_id": "OSM" } From 0051cb07c975994bbee3e00426eefcd2de8b1f5f Mon Sep 17 00:00:00 2001 From: moanos Date: Sun, 26 Jan 2025 17:06:41 +0100 Subject: [PATCH 06/10] refactor: formatting --- src/fellchensammlung/animal_shelter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fellchensammlung/animal_shelter.py b/src/fellchensammlung/animal_shelter.py index 74ba86c..9fa1357 100644 --- a/src/fellchensammlung/animal_shelter.py +++ b/src/fellchensammlung/animal_shelter.py @@ -8,12 +8,14 @@ HEADERS = { "Content-Type": "application/json" } + def load_osm_data(file_path): #Load OSM data from a GeoJSON file. with open(file_path, "r", encoding="utf-8") as file: data = json.load(file) return data + def transform_osm_data(feature): #Transform a single OSM feature into the API payload format prop = feature.get("properties", {}) @@ -30,8 +32,9 @@ def transform_osm_data(feature): "external_source_id": "OSM" } + def send_to_api(data): - #Send transformed data to the Notfellchen API. + # Send transformed data to the Notfellchen API. response = requests.post(ENDPOINT, headers=HEADERS, json=data) if response.status_code == 201: print(f"Success: Shelter '{data['name']}' uploaded.") @@ -43,7 +46,6 @@ def send_to_api(data): def main(): # Step 1: Load OSM data osm_data = load_osm_data(OSM_DATA_FILE) - # Step 2: Process each shelter and send it to the API for feature in osm_data.get("features", []): From 885bed888d2df9411f77381c04fc9a578db723fb Mon Sep 17 00:00:00 2001 From: moanos Date: Sun, 26 Jan 2025 17:07:50 +0100 Subject: [PATCH 07/10] feat: Raise connection error upon unexpected error This e.g. makes sure the API is not bombarded with unauthorized calls if the token is wrong --- src/fellchensammlung/animal_shelter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fellchensammlung/animal_shelter.py b/src/fellchensammlung/animal_shelter.py index 9fa1357..ad9ab5a 100644 --- a/src/fellchensammlung/animal_shelter.py +++ b/src/fellchensammlung/animal_shelter.py @@ -42,6 +42,8 @@ def send_to_api(data): print(f"Error: Shelter '{data['name']}' already exists or invalid data.") else: print(f"Unexpected Error: {response.status_code} - {response.text}") + raise ConnectionError + def main(): # Step 1: Load OSM data From 03a48da355c3a8816f0da120e2765812eb279544 Mon Sep 17 00:00:00 2001 From: moanos Date: Sun, 26 Jan 2025 17:59:48 +0100 Subject: [PATCH 08/10] feat: Make instance and secrets CLI argument --- src/fellchensammlung/animal_shelter.py | 59 +++++++++++++++++++++----- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/src/fellchensammlung/animal_shelter.py b/src/fellchensammlung/animal_shelter.py index ad9ab5a..472a241 100644 --- a/src/fellchensammlung/animal_shelter.py +++ b/src/fellchensammlung/animal_shelter.py @@ -1,12 +1,40 @@ +import argparse import json +import os + import requests -OSM_DATA_FILE = "osm_data.geojson" -ENDPOINT = "https://test.notfellchen.org/api/organizations/" -HEADERS = { - "Authorization": "API_KEY", - "Content-Type": "application/json" -} +DEFAULT_OSM_DATA_FILE = "osm_data.geojson" + + +def parse_args(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Upload animal shelter data to the Notfellchen API.") + parser.add_argument("--api-token", type=str, help="API token for authentication.") + parser.add_argument("--instance", type=str, help="API instance URL.") + parser.add_argument("--data-file", type=str, help="Path to the GeoJSON file containing (only) animal shelters.") + return parser.parse_args() + + +def get_config(): + """Get configuration from environment variables or command-line arguments.""" + args = parse_args() + + api_token = args.api_token or os.getenv("NOTFELLCHEN_API_TOKEN") + instance = args.instance or os.getenv("NOTFELLCHEN_INSTANCE") + data_file = args.data_file or os.getenv("NOTFELLCHEN_DATA_FILE", DEFAULT_OSM_DATA_FILE) + + if not api_token or not instance: + raise ValueError("API token and instance URL must be provided via environment variables or CLI arguments.") + + return api_token, instance, data_file + + +def load_osm_data(file_path): + """Load OSM data from a GeoJSON file.""" + with open(file_path, "r", encoding="utf-8") as file: + data = json.load(file) + return data def load_osm_data(file_path): @@ -33,9 +61,9 @@ def transform_osm_data(feature): } -def send_to_api(data): +def send_to_api(data, endpoint, headers): # Send transformed data to the Notfellchen API. - response = requests.post(ENDPOINT, headers=HEADERS, json=data) + response = requests.post(endpoint, headers=headers, json=data) if response.status_code == 201: print(f"Success: Shelter '{data['name']}' uploaded.") elif response.status_code == 400: @@ -46,13 +74,24 @@ def send_to_api(data): def main(): + # Get configuration + api_token, instance, data_file = get_config() + + # Set headers and endpoint + endpoint = f"{instance}/api/organizations/" + headers = { + "Authorization": f"Token {api_token}", + "Content-Type": "application/json" + } + # Step 1: Load OSM data - osm_data = load_osm_data(OSM_DATA_FILE) + osm_data = load_osm_data(data_file) # Step 2: Process each shelter and send it to the API for feature in osm_data.get("features", []): shelter_data = transform_osm_data(feature) - send_to_api(shelter_data) + send_to_api(shelter_data, endpoint, headers) + if __name__ == "__main__": main() From ad06829c31764b3b5f4099b96fc7817355e6ea19 Mon Sep 17 00:00:00 2001 From: moanos Date: Sun, 26 Jan 2025 18:00:23 +0100 Subject: [PATCH 09/10] feat: Add debug information --- src/fellchensammlung/animal_shelter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fellchensammlung/animal_shelter.py b/src/fellchensammlung/animal_shelter.py index 472a241..2dabe57 100644 --- a/src/fellchensammlung/animal_shelter.py +++ b/src/fellchensammlung/animal_shelter.py @@ -67,7 +67,7 @@ def send_to_api(data, endpoint, headers): if response.status_code == 201: print(f"Success: Shelter '{data['name']}' uploaded.") elif response.status_code == 400: - print(f"Error: Shelter '{data['name']}' already exists or invalid data.") + print(f"Error: Shelter '{data['name']}' already exists or invalid data. {response.text}") else: print(f"Unexpected Error: {response.status_code} - {response.text}") raise ConnectionError From ff31caa139a27b6e8d4deabf083934d1a010f3fa Mon Sep 17 00:00:00 2001 From: moanos Date: Sun, 26 Jan 2025 18:01:38 +0100 Subject: [PATCH 10/10] refactor: Move script to dedicated folder --- .../animal_shelter.py => scripts/upload_animal_shelters.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/fellchensammlung/animal_shelter.py => scripts/upload_animal_shelters.py (100%) diff --git a/src/fellchensammlung/animal_shelter.py b/scripts/upload_animal_shelters.py similarity index 100% rename from src/fellchensammlung/animal_shelter.py rename to scripts/upload_animal_shelters.py