From a65c804f6682ea567601cc4ef612cb478ac5410f Mon Sep 17 00:00:00 2001 From: Jason Cameron Date: Tue, 2 Apr 2024 18:15:01 +0000 Subject: [PATCH] Furthur optimizations --- gameserver/models/cache.py | 51 ++++- gameserver/models/contest.py | 3 + gameserver/models/problem.py | 330 ++++++++++++++++----------------- gameserver/models/profile.py | 1 + gameserver/utils/challenge.py | 22 ++- gameserver/views/contest.py | 23 ++- gameserver/views/problem.py | 8 +- gameserver/views/submission.py | 2 +- gameserver/views/user.py | 10 +- 9 files changed, 245 insertions(+), 205 deletions(-) diff --git a/gameserver/models/cache.py b/gameserver/models/cache.py index 79d9d06..11c3865 100644 --- a/gameserver/models/cache.py +++ b/gameserver/models/cache.py @@ -1,5 +1,5 @@ -from typing import TYPE_CHECKING, Optional, Self - +from typing import TYPE_CHECKING, Optional, Self, Protocol +from django.http import HttpRequest from django.apps import apps from django.db import models, transaction from django.db.models import ( @@ -8,28 +8,59 @@ Count, F, OuterRef, - Q, Subquery, Sum, Value, When, Window, ) -from django.db.models.functions import Coalesce, DenseRank, Rank, RowNumber +from django.db.models.functions import Coalesce, Rank, RowNumber -from .contest import Contest, ContestParticipation, ContestSubmission if TYPE_CHECKING: from .profile import User + from .contest import Contest, ContestParticipation, ContestSubmission + + +class ResetableCache(Protocol): + def can_reset(cls, request: HttpRequest) -> None: ... + + +class CacheMeta(models.Model): + + class Meta: + abstract = True + permissions = [ + ( + "can_reset_cache", + "Designates if the user has permission to reset the scoring caches or not.", + ) + ] + + @classmethod + def _can_reset(cls, request: HttpRequest): + return all( + [ + request.user.is_authenticated, + request.user.is_staff, + request.GET.get("reset", "") == "true", + ] + ) -class UserScore(models.Model): +class UserScore(CacheMeta): user = models.OneToOneField("User", on_delete=models.CASCADE, db_index=True) points = models.PositiveIntegerField(help_text="The amount of points.", default=0) flag_count = models.PositiveIntegerField( help_text="The amount of flags the user/team has.", default=0 ) + @classmethod + def can_reset(cls, request: HttpRequest): + return cls._can_reset(request) and request.user.has_perm( + "gameserver.can_reset_cache_user_score" + ) + def __str__(self) -> str: return self.user.username @@ -97,7 +128,7 @@ def reset_data(cls): cls.objects.bulk_create(scores_to_create) -class ContestScore(models.Model): +class ContestScore(CacheMeta): participation = models.OneToOneField( "ContestParticipation", on_delete=models.CASCADE, db_index=True ) @@ -106,6 +137,12 @@ class ContestScore(models.Model): help_text="The amount of flags the user/team has.", default=0 ) + @classmethod + def can_reset(cls, request: HttpRequest): + return cls._can_reset(request) and request.user.has_perm( + "gameserver.can_reset_cache_user_score" + ) + def get_absolute_url(self): return self.participation.get_absolute_url() diff --git a/gameserver/models/contest.py b/gameserver/models/contest.py index a4a0273..54cc64e 100644 --- a/gameserver/models/contest.py +++ b/gameserver/models/contest.py @@ -13,6 +13,7 @@ from django.urls import reverse from django.utils import timezone from django.utils.functional import cached_property +from gameserver.models.cache import ContestScore from ..templatetags.common_tags import strfdelta from . import abstract @@ -227,6 +228,8 @@ def get_editable_contests(cls, user): class ContestParticipation(models.Model): + cache = ContestScore + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.ContestScore = apps.get_model("gameserver", "ContestScore", require_ready=True) diff --git a/gameserver/models/problem.py b/gameserver/models/problem.py index 4b7cd22..64fc2e5 100644 --- a/gameserver/models/problem.py +++ b/gameserver/models/problem.py @@ -16,183 +16,183 @@ class ProblemGroup(abstract.Category): - pass + pass class ProblemType(abstract.Category): - pass + pass def gen_opaque_id(): - return secrets.token_urlsafe(32) + return secrets.token_urlsafe(32) class Problem(models.Model): - author = models.ManyToManyField("User", related_name="problems_authored", blank=True) - testers = models.ManyToManyField("User", related_name="problems_testing", blank=True) - organizations = models.ManyToManyField("Organization", related_name="problems", blank=True) - - name = models.CharField(max_length=128) - slug = models.SlugField(unique=True, db_index=True) - description = models.TextField() - summary = models.CharField(max_length=150) - - opaque_id = models.CharField(max_length=172, default=gen_opaque_id, editable=False, unique=True) - - problem_group = models.ManyToManyField(ProblemGroup, related_name="problems", blank=True) - problem_type = models.ManyToManyField(ProblemType, related_name="problems", blank=True) - - flag = models.CharField(max_length=256) - points = models.PositiveSmallIntegerField() - challenge_spec = models.JSONField(null=True, blank=True) - log_submission_content = models.BooleanField(default=False) - - is_public = models.BooleanField(default=False) - - date_created = models.DateTimeField(auto_now_add=True) - - firstblood = models.ForeignKey( - "Submission", related_name="firstblooded", null=True, blank=True, on_delete=models.PROTECT - ) - - class Meta: - permissions = ( - ("change_problem_visibility", "Change visibility of problems"), - ("edit_all_problems", "Edit all problems"), - ) - - def __str__(self): - return self.name - - def get_absolute_url(self): - return reverse("problem_detail", args=[self.slug]) - - @cached_property - def is_private(self): - return not self.is_public - - @property - def flag_format(self): - flag_format_match = re.match(r"(.*)\{.*\}", self.flag) - - if flag_format_match is not None: - return f"{flag_format_match.group(1)}{{}}" - else: - return None - - def contest_problem(self, contest): - try: - return ContestProblem.objects.get(problem=self, contest=contest) - except ContestProblem.DoesNotExist: - return None - - @cached_property - def ongoing_contests_problem(self): - now = timezone.now() - return ContestProblem.objects.filter( - problem=self, - contest__start_time__lte=now, - contest__end_time__gte=now, - ) - - def create_challenge_instance(self, instance_owner): - if self.challenge_spec is not None: - return challenge.create_challenge_instance( - self.challenge_spec, self.slug, self.flag, instance_owner - ) - - def fetch_challenge_instance(self, instance_owner): - if self.challenge_spec is not None: - return challenge.fetch_challenge_instance( - self.challenge_spec, self.slug, instance_owner - ) - - def delete_challenge_instance(self, instance_owner): - if self.challenge_spec is not None: - return challenge.delete_challenge_instance( - self.challenge_spec, self.slug, instance_owner - ) - - def is_attempted_by(self, user): - return self.submissions.filter(user=user).exists() - - def is_solved_by(self, user): - return self.submissions.filter(user=user, is_correct=True).exists() - - def is_firstblooded_by(self, user): - return self.firstblood.user == user if self.firstblood else False - - def is_accessible_by(self, user): - if self.is_public: - return True - - if not user.is_authenticated: - return False - - if self.organizations.filter(pk__in=user.organizations.all()).exists(): - return True - - if user.current_contest is not None and user.current_contest.contest.has_problem(self): - return True - - return self.is_editable_by(user) - - def is_editable_by(self, user): - if user.is_superuser or user.has_perm("gameserver.edit_all_problems"): - return True - - if user.is_authenticated: - if self.author.filter(id=user.id).exists() or self.testers.filter(id=user.id).exists(): - return True - - return False - - @classmethod - def get_public_problems(cls): - return cls.objects.filter(is_public=True) - - @classmethod - def get_visible_problems(cls, user): - if not user.is_authenticated: - return cls.get_public_problems() - - if user.is_superuser or user.has_perm("gameserver.edit_all_problems"): - return cls.objects.all() - - return cls.objects.filter( - Q(is_public=True) - | Q(author=user) - | Q(testers=user) - | Q(organizations__in=user.organizations.all()) - ).distinct() - - @classmethod - def get_editable_problems(cls, user): - if user.is_superuser or user.has_perm("gameserver.edit_all_problems"): - return cls.objects.all() - - return cls.objects.filter(author=user).distinct() + author = models.ManyToManyField("User", related_name="problems_authored", blank=True) + testers = models.ManyToManyField("User", related_name="problems_testing", blank=True) + organizations = models.ManyToManyField("Organization", related_name="problems", blank=True) + + name = models.CharField(max_length=128) + slug = models.SlugField(unique=True, db_index=True) + description = models.TextField() + summary = models.CharField(max_length=150) + + opaque_id = models.CharField(max_length=172, default=gen_opaque_id, editable=False, unique=True) + + problem_group = models.ManyToManyField(ProblemGroup, related_name="problems", blank=True) + problem_type = models.ManyToManyField(ProblemType, related_name="problems", blank=True) + + flag = models.CharField(max_length=256) + points = models.PositiveSmallIntegerField() + challenge_spec = models.JSONField(null=True, blank=True) + log_submission_content = models.BooleanField(default=False) + + is_public = models.BooleanField(default=False) + + date_created = models.DateTimeField(auto_now_add=True) + + firstblood = models.ForeignKey( + "Submission", related_name="firstblooded", null=True, blank=True, on_delete=models.PROTECT + ) + + class Meta: + permissions = ( + ("change_problem_visibility", "Change visibility of problems"), + ("edit_all_problems", "Edit all problems"), + ) + + def __str__(self): + return self.name + + def get_absolute_url(self): + return reverse("problem_detail", args=[self.slug]) + + @cached_property + def is_private(self): + return not self.is_public + + @property + def flag_format(self): + flag_format_match = re.match(r"(.*)\{.*\}", self.flag) + + if flag_format_match is not None: + return f"{flag_format_match.group(1)}{{}}" + else: + return None + + def contest_problem(self, contest): + try: + return ContestProblem.objects.get(problem=self, contest=contest) + except ContestProblem.DoesNotExist: + return None + + @cached_property + def ongoing_contests_problem(self): + now = timezone.now() + return ContestProblem.objects.filter( + problem=self, + contest__start_time__lte=now, + contest__end_time__gte=now, + ) + + def create_challenge_instance(self, instance_owner): + if self.challenge_spec is not None: + return challenge.create_challenge_instance( + self.challenge_spec, self.slug, self.flag, instance_owner + ) + + def fetch_challenge_instance(self, instance_owner): + if self.challenge_spec is not None: + return challenge.fetch_challenge_instance( + self.challenge_spec, self.slug, instance_owner + ) + + def delete_challenge_instance(self, instance_owner): + if self.challenge_spec is not None: + return challenge.delete_challenge_instance( + self.challenge_spec, self.slug, instance_owner + ) + + def is_attempted_by(self, user): + return self.submissions.filter(user=user).exists() + + def is_solved_by(self, user): + return self.submissions.filter(user=user, is_correct=True).exists() + + def is_firstblooded_by(self, user): + return self.firstblood.user == user if self.firstblood else False + + def is_accessible_by(self, user): + if self.is_public: + return True + + if not user.is_authenticated: + return False + + if self.organizations.filter(pk__in=user.organizations.all()).exists(): + return True + + if user.current_contest is not None and user.current_contest.contest.has_problem(self): + return True + + return self.is_editable_by(user) + + def is_editable_by(self, user): + if user.is_superuser or user.has_perm("gameserver.edit_all_problems"): + return True + + if user.is_authenticated: + if self.author.filter(id=user.id).exists() or self.testers.filter(id=user.id).exists(): + return True + + return False + + @classmethod + def get_public_problems(cls): + return cls.objects.filter(is_public=True) + + @classmethod + def get_visible_problems(cls, user): + if not user.is_authenticated: + return cls.get_public_problems() + + if user.is_superuser or user.has_perm("gameserver.edit_all_problems"): + return cls.objects.all() + + return cls.objects.filter( + Q(is_public=True) + | Q(author=user) + | Q(testers=user) + | Q(organizations__in=user.organizations.all()) + ).distinct() + + @classmethod + def get_editable_problems(cls, user): + if user.is_superuser or user.has_perm("gameserver.edit_all_problems"): + return cls.objects.all() + + return cls.objects.filter(author=user).distinct() def problem_file_path(instance, filename): - return f"problem/{instance.problem.opaque_id}/{filename}" + return f"problem/{instance.problem.opaque_id}/{filename}" class ProblemFile(models.Model): - problem = models.ForeignKey(Problem, on_delete=models.CASCADE, related_name="files") - artifact = models.FileField(max_length=100 + 172, upload_to=problem_file_path, unique=True) - checksum = models.CharField(max_length=64) - - def __str__(self): - return self.file_name - - @property - def file_name(self): - return self.artifact.name.split("/")[-1] - - def save(self, *args, **kwargs): - hash_sha256 = hashlib.sha256() - for chunk in self.artifact.chunks(): - hash_sha256.update(chunk) - self.checksum = hash_sha256.hexdigest() - super().save(*args, **kwargs) + problem = models.ForeignKey(Problem, on_delete=models.CASCADE, related_name="files") + artifact = models.FileField(max_length=100 + 172, upload_to=problem_file_path, unique=True) + checksum = models.CharField(max_length=64) + + def __str__(self): + return self.file_name + + @property + def file_name(self): + return self.artifact.name.split("/")[-1] + + def save(self, *args, **kwargs): + hash_sha256 = hashlib.sha256() + for chunk in self.artifact.chunks(): + hash_sha256.update(chunk) + self.checksum = hash_sha256.hexdigest() + super().save(*args, **kwargs) diff --git a/gameserver/models/profile.py b/gameserver/models/profile.py index 2d94502..0e0cbfb 100644 --- a/gameserver/models/profile.py +++ b/gameserver/models/profile.py @@ -19,6 +19,7 @@ def get_default_user_timezone(): class User(AbstractUser): + cache = UserScore full_name = models.CharField(max_length=80, blank=True) description = models.TextField(blank=True) diff --git a/gameserver/utils/challenge.py b/gameserver/utils/challenge.py index 90b9833..8e68205 100644 --- a/gameserver/utils/challenge.py +++ b/gameserver/utils/challenge.py @@ -69,12 +69,14 @@ def generate_identifier(): "apparmorProfile" ]["type"] == "Unconfined" - else f'localhost/{challenge_cluster["securityContexts"][container["securityContext"]]["apparmorProfile"]["localhostProfile"]}' - if challenge_cluster["securityContexts"][container["securityContext"]][ - "apparmorProfile" - ]["type"] - == "Localhost" - else "runtime/default" + else ( + f'localhost/{challenge_cluster["securityContexts"][container["securityContext"]]["apparmorProfile"]["localhostProfile"]}' + if challenge_cluster["securityContexts"][ + container["securityContext"] + ]["apparmorProfile"]["type"] + == "Localhost" + else "runtime/default" + ) ) for container in challenge_spec["containers"] if "securityContext" in container @@ -146,9 +148,11 @@ def generate_identifier(): "enableServiceLinks": False, "restartPolicy": "OnFailure", "runtimeClassName": challenge_cluster["runtimeClassNames"][ - challenge_spec["runtimeClassName"] - if "runtimeClassName" in challenge_spec - else "default" + ( + challenge_spec["runtimeClassName"] + if "runtimeClassName" in challenge_spec + else "default" + ) ], }, }, diff --git a/gameserver/views/contest.py b/gameserver/views/contest.py index 6fd1754..05b1a31 100644 --- a/gameserver/views/contest.py +++ b/gameserver/views/contest.py @@ -137,7 +137,7 @@ def get_context_data(self, **kwargs): context["team_participant_count"] = data top_participations = self.object.ranks() # .prefetch_related("team", "participants"), - print(top_participations.first().participation.contest) + # print(top_participations.first().participation.contest) context["top_participations"] = top_participations[:10] return context @@ -195,18 +195,12 @@ class ContestScoreboard(SingleObjectMixin, ListView, mixin.MetaMixin): model = models.ContestParticipation template_name = "contest/scoreboard.html" paginate_by = 50 - + def get_title(self): return "Scoreboard for " + self.object.name def get_queryset(self): - if all( - [ - self.request.user.is_authenticated, - self.request.user.is_staff, - self.request.GET.get("reset", "") == "true", - ] - ): + if self.model.cache.can_reset(self.request): ContestScore.reset_data(contest=self.object) return ContestScore.ranks(contest=self.object).only("team").select_related("team") @@ -238,9 +232,14 @@ def get_title(self): return self.org.short_name + " Scoreboard for " + self.contest.name def get_queryset(self): - return ContestScore.ranks( - self.contest, self.contest.participations.filter(participants__organizations=self.org) - ).only("team").select_related("team") + return ( + ContestScore.ranks( + self.contest, + self.contest.participations.filter(participants__organizations=self.org), + ) + .only("team") + .select_related("team") + ) # return self.contest._ranks( # self.contest.participations.filter(participants__organizations=self.org), # ).select_related("team") diff --git a/gameserver/views/problem.py b/gameserver/views/problem.py index 2ce08d0..a648ee4 100644 --- a/gameserver/views/problem.py +++ b/gameserver/views/problem.py @@ -33,9 +33,11 @@ def get_queryset(self): .prefetch_related("problem_type", "problem_group") .filter( Q(problem_type__in=self.selected_types) if len(self.selected_types) else Q(), - Q(problem_group__in=self.selected_groups) - if len(self.selected_groups) and not self.request.in_contest - else Q(), + ( + Q(problem_group__in=self.selected_groups) + if len(self.selected_groups) and not self.request.in_contest + else Q() + ), ) .distinct() .order_by("points", "name") diff --git a/gameserver/views/submission.py b/gameserver/views/submission.py index 69dedeb..534a34d 100644 --- a/gameserver/views/submission.py +++ b/gameserver/views/submission.py @@ -13,7 +13,7 @@ class SubmissionList(ListView, mixin.MetaMixin): def get_queryset(self): return ( models.Submission.get_visible_submissions(self.request.user) - .only("pk", "is_correct", "problem", "user", "date_created") + .only("pk", "is_correct", "problem", "user", "date_created") .select_related("user", "problem") .order_by("-pk") ) diff --git a/gameserver/views/user.py b/gameserver/views/user.py index b9f52da..89a7f32 100644 --- a/gameserver/views/user.py +++ b/gameserver/views/user.py @@ -29,13 +29,7 @@ class UserList(ListView, mixin.MetaMixin): title = "Users" def get_queryset(self): - if all( - [ - self.request.user.is_authenticated, - self.request.user.is_staff, - self.request.GET.get("reset", "") == "true", - ] - ): + if self.model.cache.can_reset(self.request): UserScore.reset_data() return UserScore.ranks() @@ -76,7 +70,7 @@ def get_queryset(self): models.Submission.get_visible_submissions(self.request.user) .filter(user=self.object) .only("pk", "is_correct", "problem", "user", "date_created") - .select_related("user", "problem") + .select_related("user", "problem") .order_by("-pk") )