Merge pull request #11 from mcpt/merge-fix

Update optimizations

.gitignore

@@ -6,3 +6,4 @@ static/**
 .build
 /.direnv
 /result
+/data.json

Dockerfile

@@ -12,7 +12,7 @@ WORKDIR /app2/media
 WORKDIR /app2/static
 WORKDIR /app
 USER ctf
-RUN python -m pip install --no-cache-dir --no-warn-script-location poetry 
+RUN python -m pip install --no-cache-dir --no-warn-script-location poetry
 
 COPY poetry.lock pyproject.toml /app/
 RUN python -m poetry config virtualenvs.in-project true && \
@@ -27,7 +27,7 @@ USER ctf
 COPY . /app/
 COPY ./mCTF/docker_config.py /app/mCTF/config.py
 USER root
-RUN set -eux; cd /app/public/scss; mkdir out; for f in *.scss; \ 
+RUN set -eux; cd /app/public/scss; mkdir out; for f in *.scss; \
     do \
       sassc --style compressed -- "$f" "out/${f%.scss}.css"; \
     done; \
@@ -41,5 +41,6 @@ EXPOSE 28730
 CMD /app/.venv/bin/gunicorn \
       --bind :28730 \
       --error-logfile - \
+      --timeout 120 \
       --config /app/container/gunicorn.py \
       mCTF.wsgi:application

gameserver/admin.py

@@ -95,6 +95,32 @@ def get_authors(self, obj):
         return ", ".join([u.username for u in obj.author.all()])
 
 
+class SubmissionAdmin(admin.ModelAdmin):
+
+    list_display = [
+        "problem",
+        "date_created",
+        "is_correct",
+        "display_firstblooded",
+    ]
+    list_filter = [
+        "problem__points",
+        "is_correct",
+        "problem",
+    ]
+    search_fields = [
+        "problem__name",
+        "user__username",
+        "user__full_name",
+    ]
+
+    def display_firstblooded(self, obj: models.Submission):
+        return obj.is_firstblood
+
+    display_firstblooded.short_description = "Firstblooded"
+    display_firstblooded.boolean = True
+        # return obj.firstblooded
+
 class OrganizationAdmin(admin.ModelAdmin):
     list_display = ["__str__", "owner", "member_count", "is_open"]
 
@@ -254,7 +280,7 @@ class UserAdmin(admin.ModelAdmin):
 admin.site.register(models.ContestScore)
 admin.site.register(models.UserScore)
 admin.site.register(models.Problem, ProblemAdmin)
-admin.site.register(models.Submission)
+admin.site.register(models.Submission, SubmissionAdmin)
 admin.site.register(models.ProblemType)
 admin.site.register(models.ProblemGroup)
 admin.site.register(models.Organization, OrganizationAdmin)

gameserver/models/contest.py

@@ -10,11 +10,11 @@
 from django.db.models.functions import Coalesce, Rank
 from django.urls import reverse
 from django.utils import timezone
+from django.utils.functional import cached_property
 
 from django.db import models
 from django.db.models import F
 from . import abstract
-from functools import cached_property 
 from .cache import ContestScore
 
 # Create your models here.
@@ -262,12 +262,10 @@ def participant(self):
 
     def _get_unique_correct_submissions(self):
         # Switch to ContestProblem -> Problem Later
-        return (
-            self.submissions.filter(submission__is_correct=True)
-            .values("submission__problem", "problem__points")
-            .distinct()
-        )
-
+        return (self.submissions.filter(submission__is_correct=True)
+                .select_related('problem')
+                .values('problem','problem__points').distinct())
+
     def points(self):
         points = self._get_unique_correct_submissions().aggregate(
             points=Coalesce(Sum("problem__points"), 0)
@@ -385,6 +383,7 @@ class ContestSubmission(models.Model):
         on_delete=models.CASCADE,
         related_name="submissions",
         related_query_name="submission",
+        db_index=True,
     )
     problem = models.ForeignKey(
         ContestProblem,
@@ -393,7 +392,8 @@ class ContestSubmission(models.Model):
         related_query_name="submission",
     )
     submission = models.OneToOneField(
-        "Submission", on_delete=models.CASCADE, related_name="contest_submission"
+        "Submission", on_delete=models.CASCADE, related_name="contest_submission",
+        db_index=True,
     )
 
     def __str__(self):
@@ -412,5 +412,9 @@ def is_firstblood(self):
         return prev_correct_submissions.count() == 1 and prev_correct_submissions.first() == self
 
     def save(self, *args, **kwargs):
+        for key in cache.get(f"contest_ranks_{self.participation.contest.pk}", default=[]):
+            cache.delete(key)
+        cache.delete(f'contest_participant_{self.participation.id}_last_solve')       # todo convert to internal django delete key due to @cachedproperty
+        cache.delete(f'contest_participant_{self.participation.id}_time_taken')
         ContestScore.invalidate(self.participation)
         super().save(*args, **kwargs)

gameserver/models/problem.py

@@ -71,7 +71,7 @@ def __str__(self):
     def get_absolute_url(self):
         return reverse("problem_detail", args=[self.slug])
 
-    @property
+    @cached_property
     def is_private(self):
         return not self.is_public
 

gameserver/models/profile.py

@@ -79,7 +79,6 @@ def rank(self, queryset=None):
         )
 
     @classmethod
-
     def ranks(cls, queryset=None):
         if queryset is None:
             queryset = cls.objects.all()

gameserver/models/submission.py

@@ -1,6 +1,7 @@
 from django.db import models
-from .cache import UserScore
 from django.db.models import Q
+from django.utils.functional import cached_property
+from .cache import UserScore
 
 class Submission(models.Model):
     user = models.ForeignKey(
@@ -29,7 +30,7 @@ def save(self, *args, **kwargs):
             UserScore.update_or_create(user=self.user, change_in_score=self.problem.points, update_flags=True)
         return super().save(*args, **kwargs)
 
-    @property
+    @cached_property
     def is_firstblood(self):
         return self.problem.firstblood == self
 

gameserver/views/contest.py

@@ -9,6 +9,7 @@
 from django.views.generic.base import RedirectView
 from django.views.generic.detail import SingleObjectMixin
 from django.views.generic.edit import CreateView, FormMixin, FormView
+from django.core.cache import cache
 
 from .. import forms, models
 from . import mixin
@@ -109,22 +110,38 @@ def form_valid(self, form):
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-        if self.request.user.is_authenticated:
-            context["user_accessible"] = self.object.is_accessible_by(self.request.user)
-            context["user_participation"] = self.request.user.participation_for_contest(self.object)
-            context["team_participant_count"] = {
-                team_pk: participant_count
-                for team_pk, participant_count in self.request.user.teams.annotate(
-                    participant_count=Count(
-                        "contest_participations__participants",
-                        filter=Q(contest_participations__contest=self.object),
-                    )
-                ).values_list("pk", "participant_count")
-            }
-        context["top_participations"] = self.object.ranks()[:10]
+        user = self.request.user
+        if user.is_authenticated:
+            if (data := cache.get(f"contest_{self.object.slug}_participant_count")) is None:
+                context["user_accessible"] = self.object.is_accessible_by(user)
+                context["user_participation"] = user.participation_for_contest(self.object)
+                team_participant_count = (
+                    user.teams.annotate(
+                        participant_count=Count(
+                            "contest_participations__participants",
+                            filter=Q(contest_participations__contest=self.object),
+                        )
+                    ).values_list("pk", "participant_count")
+                )
+                context["team_participant_count"] = {team_pk: participant_count for team_pk, participant_count in
+                                                         team_participant_count}
+                cache.set(f"contest_{self.object.slug}_participant_count", context["team_participant_count"], 60 * 3)  # Cache for 3 minutes (180 seconds) \
+            else:
+                context["team_participant_count"] = data
+
+        top_participations = cache_this(self.object.ranks().prefetch_related("participants", "team"), f"contest_{self.object.slug}_ranks", 60 * 5)  # Cache for 5 minutes (300 seconds)
+        context["top_participations"] = top_participations[:10]
+
         return context
 
 
+def cache_this(queryset, cache_key: str, timeout: int = 60 * 5):
+    from django.db import connection
+    data = cache.get(cache_key)
+    if not data:
+        data = list(queryset)
+        cache.set(cache_key, data, timeout)
+    return data
 @method_decorator(require_POST, name="dispatch")
 class ContestLeave(LoginRequiredMixin, RedirectView):
     pattern_name = "contest_detail"
@@ -182,18 +199,16 @@ def get_queryset(self):
         queryset = cache.get(cache_key)
         if not queryset or self.request.GET.get('cache_reset', '').casefold() == "yaaaa":
             queryset = self.object.ranks().prefetch_related('team', 'submissions__problem')
-            cache.set(cache_key, queryset, 5 * 5)  # Cache for 5 minutes (300 seconds)
+            cache.set(cache_key, queryset, 60 * 5)  # Cache for 5 minutes (300 seconds)
         return queryset
 
-    @staticmethod
-    def _get_contest(slug):
-        cache_key = f"contest_{slug}_scoreboard_contest"
-        contest = cache.get(cache_key)
-        if not contest or self.request.GET.get('cache_reset', '').casefold() == "yaaaa":
-            contest = get_object_or_404(models.Contest, slug=slug)
-            cache.set(cache_key, contest, 5 * 5)  # Cache for 5 minutes (300 seconds)
-        return contest
-
+    def _get_contest(self, slug):
+        # cache_key = f"contest_{slug}_scoreboard_contest"
+        # contest = cache.get(cache_key)
+        # if not contest or self.request.GET.get('cache_reset', '').casefold() == "yaaaa":
+        contest = get_object_or_404(models.Contest, slug=slug)
+            # cache.set(cache_key, contest, 60 * 5)  # Cache for 5 minutes (300 seconds)
+
     def get(self, request, *args, **kwargs):
         self.object = self._get_contest(self.kwargs["slug"])
         return super().get(request, *args, **kwargs)
@@ -239,15 +254,15 @@ def get_title(self):
 
     def get_description(self):
         return self.object.__str__()
-
+    
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-
+        
         context["recent_contest_submissions"] = self.object.submissions.order_by("-pk")[:10]
-
+        
         contest_problems = self.object.contest.problems
         participant_submissions = self.object._get_unique_correct_submissions()
-
+        
         context["problem_types"] = {
             ptype: {
                 "total": ptype.pc,
@@ -265,15 +280,15 @@ def get_context_data(self, **kwargs):
                 ),
             )
         }
-
+        
         if pus := contest_problems.filter(problem__problem_type=None):
             context["problem_types"]["Other"] = {
                 "total": pus.count(),
                 "solved": participant_submissions.filter(
                     problem__problem__problem_type=None
                 ).count(),
             }
-
+        
         # new queries instead of summation in case a problem has multiple problem_types
         context["problem_types_total"] = {
             "total": contest_problems.count(),

gameserver/views/user.py

@@ -8,6 +8,7 @@
 from django.views.generic.base import RedirectView
 from django.views.generic.detail import SingleObjectMixin
 from django.views.generic.edit import UpdateView
+from django.core.cache import cache
 
 from .. import forms, models
 from . import mixin
@@ -27,11 +28,16 @@ class UserList(ListView, mixin.MetaMixin):
     model = models.User
     template_name = "user/list.html"
     context_object_name = "users"
-    paginate_by = 40
+    paginate_by = 35
     title = "Users"
 
     def get_queryset(self):
-        return models.User.ranks()
+        cache_key = f"users_page_global_cache"
+        queryset = cache.get(cache_key)
+        if not queryset or self.request.GET.get('cache_reset', '').casefold() == "yaaaa":
+            queryset = models.User.ranks()
+            cache.set(cache_key, queryset, 10 * 60)  # Cache for 10 minutes (600 seconds)
+        return queryset
 
     def get(self, request, *args, **kwargs):
         if request.in_contest:

mCTF/settings.py

@@ -62,9 +62,6 @@
     "gameserver.middleware.ContestMiddleware",
     "django.contrib.flatpages.middleware.FlatpageFallbackMiddleware",
     "gameserver.middleware.RedirectFallbackTemporaryMiddleware",
-    'django.middleware.cache.UpdateCacheMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'django.middleware.cache.FetchFromCacheMiddleware',
     'gameserver.middleware.ErrorLogMiddleware',
     # ↑ keep last to log errors from middlewares
 ]
@@ -318,3 +315,11 @@
     raise TypeError("DEFAULT_FILE_STORAGE must not be blank")
 if STATICFILES_STORAGE == "":
     raise TypeError("STATICFILES_STORAGE must not be blank")
+
+INSTALLED_APPS += ["debug_toolbar"]
+
+MIDDLEWARE += [
+    # ...
+    "debug_toolbar.middleware.DebugToolbarMiddleware",
+    # ...
+]

mCTF/urls.py

@@ -18,6 +18,7 @@
 from django.urls import include, path
 
 urlpatterns = [
+    path("__debug__/", include("debug_toolbar.urls")),
     path("admin/", admin.site.urls),
     path("", include("gameserver.urls")),
     path("accounts/", include("allauth.urls")),