fix: massively reduce number of db queries by caching num_per_sex #27

notfellchen.cfg

@@ -8,6 +8,7 @@ host=localhost
 [django]
 secret=CHANGE-ME
 debug=True
+internal_ips=["127.0.0.1"]
 
 [database]
 backend=sqlite3

pyproject.toml

@@ -39,7 +39,7 @@ dependencies = [
     "drf-spectacular[sidecar]",
     "django-widget-tweaks",
     "django-super-deduper",
-    "django-allauth[mfa]"
+    "django-allauth[mfa]",
 ]
 
 dynamic = ["version", "readme"]
@@ -49,6 +49,7 @@ develop = [
     "pytest",
     "coverage",
     "model_bakery",
+    "debug_toolbar",
 ]
 docs = [
     "sphinx",

src/fellchensammlung/admin.py

@@ -164,6 +164,13 @@ class SocialMediaPostAdmin(admin.ModelAdmin):
     list_filter = ("platform",)
 
 
+@admin.register(Log)
+class LogAdmin(admin.ModelAdmin):
+    ordering = ["-created_at"]
+    list_filter = ("action",)
+    list_display = ("action", "user", "created_at")
+
+
 admin.site.register(Animal)
 admin.site.register(Species)
 admin.site.register(Rule)
@@ -172,5 +179,4 @@ admin.site.register(ModerationAction)
 admin.site.register(Language)
 admin.site.register(Announcement)
 admin.site.register(Subscriptions)
-admin.site.register(Log)
 admin.site.register(Timestamp)

src/fellchensammlung/forms.py

@@ -1,4 +1,5 @@
 from django import forms
+from django.forms.widgets import Textarea
 
 from .models import AdoptionNotice, Animal, Image, ReportAdoptionNotice, ReportComment, ModerationAction, User, Species, \
     Comment, SexChoicesWithAll, DistanceChoices, SpeciesSpecificURL, RescueOrganization
@@ -9,6 +10,8 @@ from django.utils.translation import gettext_lazy as _
 from notfellchen.settings import MEDIA_URL
 from crispy_forms.layout import Div
 
+from .tools.model_helpers import reason_for_signup_label, reason_for_signup_help_text
+
 
 def animal_validator(value: str):
     value = value.lower()
@@ -138,8 +141,12 @@ class ModerationActionForm(forms.ModelForm):
 
 
 class AddedRegistrationForm(forms.Form):
+    reason_for_signup = forms.CharField(label=reason_for_signup_label,
+                                        help_text=reason_for_signup_help_text,
+                                        widget=Textarea)
     captcha = forms.CharField(validators=[animal_validator], label=_("Nenne eine bekannte Tierart"), help_text=_(
-        "Bitte nenne hier eine bekannte Tierart (z.B. ein Tier das an der Leine geführt wird). Das Fragen wir dich um sicherzustellen, dass du kein Roboter bist."))
+        "Bitte nenne hier eine bekannte Tierart (z.B. ein Tier das an der Leine geführt wird). Das Fragen wir dich um "
+        "sicherzustellen, dass du kein Roboter bist."))
 
     def signup(self, request, user):
         pass

src/fellchensammlung/models.py

@@ -13,7 +13,8 @@ from notfellchen.settings import MEDIA_URL, base_url
 from .tools.geo import LocationProxy, Position
 from .tools.misc import time_since_as_hr_string
 from .tools.model_helpers import NotificationTypeChoices, AdoptionNoticeStatusChoices, AdoptionProcess, \
-    AdoptionNoticeStatusChoicesDescriptions, RegularCheckStatusChoices
+    AdoptionNoticeStatusChoicesDescriptions, RegularCheckStatusChoices, reason_for_signup_label, \
+    reason_for_signup_help_text
 from .tools.model_helpers import ndm as NotificationDisplayMapping
 
 
@@ -267,10 +268,6 @@ class RescueOrganization(models.Model):
         """
         return self.instagram or self.facebook or self.website or self.phone_number or self.email or self.fediverse_profile
 
-    def set_exclusion_from_checks(self):
-        self.exclude_from_check = True
-        self.save()
-
     @property
     def child_organizations(self):
         return RescueOrganization.objects.filter(parent_org=self)
@@ -311,8 +308,7 @@ class User(AbstractUser):
     updated_at = models.DateTimeField(auto_now=True)
     organization_affiliation = models.ForeignKey(RescueOrganization, on_delete=models.PROTECT, null=True, blank=True,
                                                  verbose_name=_('Organisation'))
-    reason_for_signup = models.TextField(verbose_name=_("Grund für die Registrierung"), help_text=_(
+    reason_for_signup = models.TextField(verbose_name=reason_for_signup_label, help_text=reason_for_signup_help_text)
-        "Wir würden gerne wissen warum du dich registriertst, ob du dich z.B. Tiere eines bestimmten Tierheim einstellen willst 'nur mal gucken' willst. Beides ist toll! Wenn du für ein Tierheim/eine Pflegestelle arbeitest kontaktieren wir dich ggf. um dir erweiterte Rechte zu geben."))
     email_notifications = models.BooleanField(verbose_name=_("Benachrichtigung per E-Mail"), default=True)
     REQUIRED_FIELDS = ["reason_for_signup", "email"]
 
@@ -423,9 +419,10 @@ class AdoptionNotice(models.Model):
 
     @property
     def num_per_sex(self):
+        print(f"{self.pk} x")
         num_per_sex = dict()
         for sex in SexChoices:
-            num_per_sex[sex] = self.animals.filter(sex=sex).count()
+            num_per_sex[sex] = len([animal for animal in self.animals if animal.sex == sex])
         return num_per_sex
 
     @property
@@ -515,6 +512,7 @@ class AdoptionNotice(models.Model):
                 photos.extend(animal.photos.all())
             if len(photos) > 0:
                 return photos
+            return None
 
     def get_photo(self):
         """

src/fellchensammlung/templates/fellchensammlung/base.html

@@ -37,6 +37,26 @@
 {% block header %}
     {% include "fellchensammlung/header.html" %}
 {% endblock %}
+{% if profile %}
+    <div class="profile">
+        <table class="table is-bordered is-fullwidth is-hoverable is-striped">
+            <thead>
+            <tr>
+                <td>Timestamp</td>
+                <td>Status</td>
+            </tr>
+            </thead>
+            <tbody>
+            {% for status in profile %}
+                <tr>
+                    <td>{{ status.0 }}</td>
+                    <td>{{ status.1 }}</td>
+                </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endif %}
 <div class="main-content">
     {% block content %}{% endblock %}
 </div>
@@ -45,5 +65,8 @@
 {% block footer %}
     {% include "fellchensammlung/footer.html" %}
 {% endblock %}
+
+{% block extra_body %}
+{% endblock extra_body %}
 </body>
 </html>

src/fellchensammlung/templates/fellchensammlung/partials/sex-overview.html

@@ -1,9 +1,10 @@
 {% load static %}
 {% load i18n %}
 <div class="grid is-col-min-2">
-    {% if adoption_notice.num_per_sex.F > 0 %}
+    {% with num_per_sex=adoption_notice.num_per_sex %}
+        {% if num_per_sex.F > 0 %}
             <span class="cell icon-text tag is-medium">
-            <span class="has-text-weight-bold is-size-4">{{ adoption_notice.num_per_sex.F }} </span>
+            <span class="has-text-weight-bold is-size-4">{{ num_per_sex.F }}</span>
             <span class="icon">
                 <img class="icon" src="{% static 'fellchensammlung/img/sexes/Female.png' %}"
                      alt="{% translate 'weibliche Tiere' %}">
@@ -11,9 +12,9 @@
         </span>
         {% endif %}
 
-    {% if adoption_notice.num_per_sex.I > 0 %}
+        {% if num_per_sex.I > 0 %}
             <span class="cell icon-text tag is-medium">
-            <span class="has-text-weight-bold is-size-4">{{ adoption_notice.num_per_sex.I }}</span>
+            <span class="has-text-weight-bold is-size-4">{{ num_per_sex.I }}</span>
 
             <span class="icon">
                 <img class="icon"
@@ -22,18 +23,18 @@
             </span>
         </span>
         {% endif %}
-    {% if adoption_notice.num_per_sex.M > 0 %}
+        {% if num_per_sex.M > 0 %}
             <span class="cell icon-text tag is-medium">
-            <span class="has-text-weight-bold is-size-4">{{ adoption_notice.num_per_sex.M }}</span>
+            <span class="has-text-weight-bold is-size-4">{{ num_per_sex.M }}</span>
             <span class="icon">
                 <img class="icon" src="{% static 'fellchensammlung/img/sexes/Male.png' %}"
                      alt="{% translate 'männliche Tiere' %}">
             </span>
         </span>
         {% endif %}
-    {% if adoption_notice.num_per_sex.M_N > 0 %}
+        {% if num_per_sex.M_N > 0 %}
             <span class="cell icon-text tag is-medium">
-            <span class="has-text-weight-bold is-size-4">{{ adoption_notice.num_per_sex.M_N }}</span>
+            <span class="has-text-weight-bold is-size-4">{{ num_per_sex.M_N }}</span>
             <span class="icon">
                 <img class="icon"
                      src="{% static 'fellchensammlung/img/sexes/Male Neutered.png' %}"
@@ -41,4 +42,5 @@
             </span>
         </span>
         {% endif %}
+    {% endwith %}
 </div>

src/fellchensammlung/tools/misc.py

@@ -1,5 +1,6 @@
 import datetime as datetime
 import logging
+import time
 
 from django.utils.translation import ngettext
 from django.utils.translation import gettext as _
@@ -75,3 +76,20 @@ def is_404(url):
         return result.status_code == 404
     except requests.RequestException as e:
         logging.warning(f"Request to {url} failed: {e}")
+
+
+class RequestProfiler:
+    data = []
+
+    def add_status(self, status):
+        self.data.append((time.time(), status))
+
+    @property
+    def as_relative(self):
+        first_ts = self.data[0][0]
+        return [(datum[0] - first_ts, datum[1]) for datum in self.data]
+
+    @property
+    def as_relative_with_ms(self):
+        first_ts = self.data[0][0]
+        return [(f"{(datum[0] - first_ts)*1000:.4}ms", datum[1]) for datum in self.data]

src/fellchensammlung/tools/model_helpers.py

@@ -134,3 +134,14 @@ class RegularCheckStatusChoices(models.TextChoices):
     EXCLUDED_OTHER_ORG = "excluded_other_org", _("Exkludiert: Andere Organisation wird geprüft")
     EXCLUDED_SCOPE = "excluded_scope", _("Exkludiert: Organisation hat nie Notfellchen-relevanten Vermittlungen")
     EXCLUDED_OTHER = "excluded_other", _("Exkludiert: Anderer Grund")
+
+
+##########
+## USER ##
+##########
+
+reason_for_signup_label = _("Grund für die Registrierung")
+reason_for_signup_help_text = _(
+    "Wir würden gerne wissen warum du dich registrierst, ob du dich z.B. Tiere eines bestimmten Tierheim einstellen "
+    "willst 'nur mal gucken' willst. Beides ist toll! Wenn du für ein Tierheim/eine Pflegestelle arbeitest "
+    "kontaktieren wir dich ggf. um dir erweiterte Rechte zu geben.")

src/fellchensammlung/views.py

@@ -1,5 +1,4 @@
 import logging
-from datetime import timedelta
 
 from django.contrib.auth.views import redirect_to_login
 from django.core.paginator import Paginator
@@ -37,6 +36,7 @@ from .tools.admin import clean_locations, get_unchecked_adoption_notices, deacti
 from .tasks import post_adoption_notice_save
 from rest_framework.authtoken.models import Token
 
+from .tools.misc import RequestProfiler
 from .tools.model_helpers import AdoptionNoticeStatusChoices, AdoptionNoticeProcessTemplates, RegularCheckStatusChoices
 from .tools.search import AdoptionNoticeSearch, RescueOrgSearch
 
@@ -242,11 +242,17 @@ def search_important_locations(request, important_location_slug):
 
 
 def search(request, templatename="fellchensammlung/search.html"):
+    search_profile = RequestProfiler()
+    search_profile.add_status("Start")
+
     # A user just visiting the search site did not search, only upon completing the search form a user has really
     # searched. This will toggle the "subscribe" button
     searched = False
+    search_profile.add_status("Init Search")
     search = AdoptionNoticeSearch()
+    search_profile.add_status("Search from request starting")
     search.adoption_notice_search_from_request(request)
+    search_profile.add_status("Search from request finished")
     if request.method == 'POST':
         searched = True
         if "subscribe_to_search" in request.POST:
@@ -266,10 +272,12 @@ def search(request, templatename="fellchensammlung/search.html"):
         subscribed_search = search.get_subscription_or_none(request.user)
     else:
         subscribed_search = None
+    search_profile.add_status("End of POST")
     site_title = _("Suche")
     site_description = _("Ratten in Tierheimen und Rattenhilfen in der Nähe suchen.")
     canonical_url = reverse("search")
 
+    search_profile.add_status("Start of context")
     context = {"adoption_notices": search.get_adoption_notices(),
                "search_form": search.search_form,
                "place_not_found": search.place_not_found,
@@ -286,6 +294,10 @@ def search(request, templatename="fellchensammlung/search.html"):
                "site_title": site_title,
                "site_description": site_description,
                "canonical_url": canonical_url}
+    search_profile.add_status("End of context")
+    if request.user.is_superuser:
+        context["profile"] = search_profile.as_relative_with_ms
+    search_profile.add_status("Finished - returing render")
     return render(request, templatename, context=context)
 
 
@@ -578,12 +590,20 @@ def report_detail_success(request, report_id):
 
 
 def user_detail(request, user, token=None):
+    user_detail_profile = RequestProfiler()
+    user_detail_profile.add_status("Start")
+    adoption_notices = AdoptionNotice.objects.filter(owner=user)
+    user_detail_profile.add_status("Finished fetching adoption notices")
     context = {"user": user,
-               "adoption_notices": AdoptionNotice.objects.filter(owner=user),
+               "adoption_notices": adoption_notices,
                "notifications": Notification.objects.filter(user_to_notify=user, read=False),
                "search_subscriptions": SearchSubscription.objects.filter(owner=user), }
+    user_detail_profile.add_status("End of context")
     if token is not None:
         context["token"] = token
+    user_detail_profile.add_status("Finished - returning to renderer")
+    if request.user.is_superuser:
+        context["profile"] = user_detail_profile.as_relative_with_ms
     return render(request, 'fellchensammlung/details/detail-user.html', context=context)
 
 
@@ -846,8 +866,7 @@ def rescue_organization_check(request, context=None):
         action = request.POST.get("action")
         if action == "checked":
             rescue_org.set_checked()
-        elif action == "exclude":
+            Log.objects.create(user=request.user, action="rescue_organization_checked", )
-            rescue_org.set_exclusion_from_checks()
         elif action == "set_species_url":
             species_url_form = SpeciesURLForm(request.POST)
 
@@ -858,6 +877,7 @@ def rescue_organization_check(request, context=None):
         elif action == "update_internal_comment":
             comment_form = RescueOrgInternalComment(request.POST, instance=rescue_org)
             if comment_form.is_valid():
+                Log.objects.create(user=request.user, action="rescue_organization_added_internal_comment", )
                 comment_form.save()
 
     rescue_orgs_to_check = RescueOrganization.objects.filter(exclude_from_check=False,
@@ -906,6 +926,10 @@ def exclude_from_regular_check(request, rescue_organization_id, source="organiza
             to_be_excluded = form.cleaned_data["regular_check_status"] != RegularCheckStatusChoices.REGULAR_CHECK
             rescue_org.exclude_from_check = to_be_excluded
             rescue_org.save()
+            if to_be_excluded:
+                Log.objects.create(user=request.user,
+                                   action="rescue_organization_excluded_from_check",
+                                   text=f"New status: {form.cleaned_data["regular_check_status"]}")
 
             return redirect(reverse(source))
     else:

src/notfellchen/settings.py

@@ -9,7 +9,7 @@ https://docs.djangoproject.com/en/3.2/topics/settings/
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.2/ref/settings/
 """
-
+import json
 from pathlib import Path
 import os
 import configparser
@@ -74,6 +74,10 @@ except configparser.NoSectionError:
     raise BaseException("No config found or no Django Secret is configured!")
 DEBUG = config.getboolean('django', 'debug', fallback=False)
 
+# Internal IPs
+raw_config_value = config.get("django", "internal_ips", fallback=[])
+INTERNAL_IPS = json.loads(raw_config_value)
+
 """ DATABASE """
 DB_BACKEND = config.get("database", "backend", fallback="sqlite3")
 DB_NAME = config.get("database", "name", fallback="notfellchen.sqlite3")
@@ -85,6 +89,7 @@ DB_HOST = config.get("database", "host", fallback='')
 BASE_DIR = Path(__file__).resolve().parent.parent
 LOCALE_PATHS = [os.path.join(BASE_DIR, 'locale')]
 
+
 """ CELERY + KEYDB """
 CELERY_BROKER_URL = config.get("celery", "broker", fallback="redis://localhost:6379/0")
 CELERY_RESULT_BACKEND = config.get("celery", "backend", fallback="redis://localhost:6379/0")
@@ -141,7 +146,7 @@ AUTHENTICATION_BACKENDS = [
 ACCOUNT_EMAIL_VERIFICATION = "mandatory"
 ACCOUNT_EMAIL_VERIFICATION_BY_CODE_ENABLED = True
 
-ACCOUNT_SIGNUP_FIELDS = ['username*', "email*", "password1*", "password2*"]
+ACCOUNT_SIGNUP_FIELDS = ['username*', "email*", "password1*"]
 
 ACCOUNT_SIGNUP_FORM_CLASS = 'fellchensammlung.forms.AddedRegistrationForm'
 
@@ -229,10 +234,12 @@ INSTALLED_APPS = [
     'drf_spectacular',
     'drf_spectacular_sidecar',  # required for Django collectstatic discovery
     'widget_tweaks',
+    "debug_toolbar",
 ]
 
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
+    "debug_toolbar.middleware.DebugToolbarMiddleware",
     # Static file serving & caching
     'whitenoise.middleware.WhiteNoiseMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',

src/notfellchen/urls.py

@@ -18,13 +18,14 @@ from django.conf.urls.i18n import i18n_patterns
 from django.contrib import admin
 from django.urls import include, path
 from django.conf import settings
-
 from django.conf.urls.static import static
 
+from debug_toolbar.toolbar import debug_toolbar_urls
+
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('accounts/', include('allauth.urls')),
-]
+] + debug_toolbar_urls()
 
 urlpatterns += i18n_patterns(
     path("", include("fellchensammlung.urls")),