Improvements

sync
Laurent 12 months ago
parent 1f4687f78a
commit 3251dc3531
  1. 138
      api/sync.py
  2. 2
      api/utils.py
  3. 2
      api/views.py
  4. 1
      padelclub_backend/settings_app.py
  5. 54
      tournaments/admin.py
  6. 23
      tournaments/migrations/0101_remove_modellog_user_modellog_users.py
  7. 17
      tournaments/migrations/0102_remove_dataaccess_last_hierarchy_update.py
  8. 15
      tournaments/models/data_access.py
  9. 2
      tournaments/models/model_log.py
  10. 172
      tournaments/signals.py

@ -10,6 +10,7 @@ from django.db.models import Q
from django.core.exceptions import ObjectDoesNotExist
from collections import defaultdict
from urllib.parse import unquote
from .utils import build_serializer_class, get_data, get_serialized_data
@ -38,14 +39,19 @@ class DataApi(APIView):
instance = get_data('tournaments', model_name, data_id)
if model_operation == 'DELETE':
parent_model, parent_id = instance.get_parent_reference()
return self.delete_and_save_log(request, data_id, model_operation, model_name, store_id, now)
# parent_model, parent_id = instance.get_parent_reference()
instance = get_data('tournaments', model_name, data_id)
instance.delete()
return Response(status=status.HTTP_204_NO_CONTENT)
# return self.delete_and_save_log(request, data_id, model_operation, model_name, store_id, now)
else: # PUT
serializer = serializer_class(instance, data=data, context={'request': request})
if serializer.is_valid():
if instance.last_update <= serializer.validated_data.get('last_update'):
print('>>> update')
return self.save_and_create_log(request, serializer, model_operation, model_name, store_id, now)
serializer.save()
return Response(serializer.data, status=status.HTTP_200_OK)
# return self.save_and_create_log(request, serializer, model_operation, model_name, store_id, now)
else:
print('>>> return 203')
return Response(serializer.data, status=status.HTTP_203_NON_AUTHORITATIVE_INFORMATION)
@ -55,89 +61,36 @@ class DataApi(APIView):
print('>>> insert')
serializer = serializer_class(data=data, context={'request': request})
if serializer.is_valid():
return self.save_and_create_log(request, serializer, model_operation, model_name, store_id, now)
else:
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def save_and_create_log(self, request, serializer, model_operation, model_name, store_id, date):
instance = serializer.save()
self.save_model_log(
user=request.user,
model_operation=model_operation,
model_name=model_name,
model_id=instance.id,
store_id=store_id,
date=date
)
self.update_linked_data_access(instance, date)
serializer.save()
return Response(serializer.data, status=status.HTTP_201_CREATED)
def delete_and_save_log(self, request, data_id, model_operation, model_name, store_id, date):
instance = get_data('tournaments', model_name, data_id)
self.update_linked_data_access(instance, date)
instance.delete()
# We delete all previous logs linked to the instance because they won't be needed anymore
ModelLog.objects.filter(model_id=data_id).delete()
self.save_model_log(
user=request.user,
model_operation=model_operation,
model_name=model_name,
model_id=data_id,
store_id=store_id,
date=date
)
return Response(status=status.HTTP_204_NO_CONTENT)
def update_linked_data_access(self, instance, date):
related_instances = instance.related_instances()
related_ids = [ri.id for ri in instance.related_instances()]
related_ids.append(instance.id)
data_access_list = DataAccess.objects.filter(model_id__in=related_ids)
for data_access in data_access_list:
data_access.last_hierarchy_update = date
data_access.save()
def save_model_log(self, user, model_operation, model_name, model_id, store_id, date):
existing_log = ModelLog.objects.filter(user=user, model_id=model_id, operation=model_operation).first()
if existing_log:
existing_log.date = date
existing_log.model_operation = model_operation
existing_log.save()
# return self.save_and_create_log(request, serializer, model_operation, model_name, store_id, now)
else:
model_log = ModelLog()
model_log.user = user
model_log.operation = model_operation
model_log.date = date
model_log.model_name = model_name
model_log.model_id = model_id
model_log.store_id = store_id
model_log.save()
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def get(self, request, *args, **kwargs):
last_update = request.query_params.get('last_update')
if not last_update:
last_update_str = request.query_params.get('last_update')
decoded_last_update = unquote(last_update_str) # Decodes %2B into +
# print(f'last_update_str = {last_update_str}')
# print(f'decoded_last_update = {decoded_last_update}')
if not decoded_last_update:
return Response({"error": "last_update parameter is required"}, status=status.HTTP_400_BAD_REQUEST)
try:
last_update = timezone.datetime.fromisoformat(decoded_last_update)
except ValueError:
return Response({"error": f"Invalid date format for last_update: {decoded_last_update}"}, status=status.HTTP_400_BAD_REQUEST)
print(f'/data GET: {last_update}')
logs = self.query_model_logs(last_update, request.user)
print(f'>>> log count = {len(logs)}')
updates = defaultdict(dict)
deletions = defaultdict(list)
# print(f'>>> log count = {len(logs)}')
last_log_date = None
for log in logs:
last_log_date = log.date
try:
if log.operation in ['POST', 'PUT']:
data = get_serialized_data('tournaments', log.model_name, log.model_id)
@ -156,7 +109,7 @@ class DataApi(APIView):
# instance = model.objects.get(id=log.model_id)
self.add_children_recursively(instance, updates)
self.add_parents_recursively(instance, updates)
elif log.operation == 'delete data access signal':
elif log.operation == 'REVOKE_ACCESS':
print(f'revoke access {log.model_id} - {log.store_id}')
deletions[log.model_name].append({'model_id': log.model_id, 'store_id': log.store_id})
except ObjectDoesNotExist:
@ -172,38 +125,37 @@ class DataApi(APIView):
response_data = {
"updates": dict(updates),
"deletions": dict(deletions)
"deletions": dict(deletions),
"date": last_log_date
}
return Response(response_data, status=status.HTTP_200_OK)
def query_model_logs(self, last_update, user):
try:
last_update = timezone.datetime.fromisoformat(last_update)
except ValueError:
return Response({"error": f"Invalid date format for last_update: {last_update}"}, status=status.HTTP_400_BAD_REQUEST)
# print(f'last_update = {last_update}')
log_query = Q(date__gt=last_update) & Q(users=user)
return ModelLog.objects.filter(log_query).order_by('date')
# get recently modified DataAccess
data_access_query = Q(last_hierarchy_update__gt=last_update) & (Q(shared_with__in=[user]) | Q(owner=user))
data_access_list = DataAccess.objects.filter(data_access_query) #.values_list('model_id', flat=True)
# data_access_query = Q(last_hierarchy_update__gt=last_update) & (Q(shared_with__in=[user]) | Q(owner=user))
# data_access_list = DataAccess.objects.filter(data_access_query) #.values_list('model_id', flat=True)
# print(f'>> da count = {len(data_access_list)}')
# get ids of all recently updated related instances of each shared data
model_ids = []
for data_access in data_access_list:
model_ids.append(data_access.model_id)
try:
instance = get_data('tournaments', data_access.model_name, data_access.model_id)
related_instances = instance.related_instances()
related_ids = [ri.id for ri in instance.related_instances() if ri.last_update > last_update]
model_ids.extend(related_ids)
except ObjectDoesNotExist:
pass
# # get ids of all recently updated related instances of each shared data
# model_ids = []
# for data_access in data_access_list:
# model_ids.append(data_access.model_id)
# try:
# instance = get_data('tournaments', data_access.model_name, data_access.model_id)
# related_instances = instance.related_instances()
# related_ids = [ri.id for ri in instance.related_instances() if ri.last_update > last_update]
# model_ids.extend(related_ids)
# except ObjectDoesNotExist:
# pass
# get all ModelLog list since the last_update, from the user and from the data he has access to
log_query = Q(date__gt=last_update) & (Q(user=user) | Q(model_id__in=model_ids))
return ModelLog.objects.filter(log_query).order_by('date')
def add_children_recursively(self, instance, updates):
"""

@ -9,7 +9,7 @@ def is_valid_email(email):
def build_serializer_class(source):
# Remove the 's' character at the end if present
if source.endswith('s'):
if source.endswith('s') and not source.endswith('ss'):
source = source[:-1]
# Capitalize words separated by a dash

@ -313,5 +313,5 @@ class DataAccessViewSet(viewsets.ModelViewSet):
def get_queryset(self):
if self.request.user:
return self.queryset.filter(owner=self.request.user)
return self.queryset.filter(Q(owner=self.request.user) | Q(shared_with__in=[self.request.user]))
return []

@ -1,6 +1,7 @@
# Rest Framework configuration
REST_FRAMEWORK = {
'DATETIME_FORMAT': "%Y-%m-%dT%H:%M:%S%z",
# Use Django's standard `django.contrib.auth` permissions,
# or allow read-only access for unauthenticated users.
'DEFAULT_PERMISSION_CLASSES': [

@ -1,10 +1,11 @@
from django.contrib import admin
from django.utils import timezone
from tournaments.models import team_registration
from tournaments.models.data_access import DataAccess
from tournaments.models.device_token import DeviceToken
from .models import Club, TeamScore, Tournament, CustomUser, Event, Round, GroupStage, Match, TeamRegistration, PlayerRegistration, Purchase, Court, DateInterval, FailedApiCall, Log, ModelLog
from .models import BaseModel, Club, TeamScore, Tournament, CustomUser, Event, Round, GroupStage, Match, TeamRegistration, PlayerRegistration, Purchase, Court, DateInterval, FailedApiCall, Log, ModelLog
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.forms import UserCreationForm, UserChangeForm
@ -12,6 +13,12 @@ from .forms import CustomUserCreationForm, CustomUserChangeForm
from .filters import TeamScoreTournamentListFilter, MatchTournamentListFilter, SimpleTournamentListFilter, MatchTypeListFilter, SimpleIndexListFilter
class AutoUpdateAdmin(admin.ModelAdmin):
def save_model(self, request, obj, form, change):
if isinstance(obj, BaseModel):
obj.last_updated_by = request.user
obj.last_update = timezone.now()
super().save_model(request, obj, form, change)
class CustomUserAdmin(UserAdmin):
form = CustomUserChangeForm
@ -37,72 +44,77 @@ class CustomUserAdmin(UserAdmin):
),
]
class EventAdmin(admin.ModelAdmin):
class EventAdmin(AutoUpdateAdmin):
list_display = ['creation_date', 'name', 'club', 'creator']
list_filter = ['creator']
ordering = ['-creation_date']
class TournamentAdmin(admin.ModelAdmin):
class TournamentAdmin(AutoUpdateAdmin):
list_display = ['display_name', 'event', 'is_private', 'start_date', 'payment', 'is_canceled']
list_filter = ['is_deleted']
class TeamRegistrationAdmin(admin.ModelAdmin):
class TeamRegistrationAdmin(AutoUpdateAdmin):
list_display = ['player_names', 'group_stage_position', 'name', 'tournament']
list_filter = [SimpleTournamentListFilter]
class TeamScoreAdmin(admin.ModelAdmin):
class TeamScoreAdmin(AutoUpdateAdmin):
list_display = ['team_registration', 'score', 'walk_out', 'match']
list_filter = [TeamScoreTournamentListFilter]
class RoundAdmin(admin.ModelAdmin):
class RoundAdmin(AutoUpdateAdmin):
list_display = ['tournament', 'name', 'index', 'parent']
list_filter = [SimpleTournamentListFilter, SimpleIndexListFilter]
search_fields = ['id']
class PlayerRegistrationAdmin(admin.ModelAdmin):
class PlayerRegistrationAdmin(AutoUpdateAdmin):
list_display = ['first_name', 'last_name', 'licence_id', 'rank']
search_fields = ('first_name', 'last_name')
class MatchAdmin(admin.ModelAdmin):
class MatchAdmin(AutoUpdateAdmin):
list_display = ['__str__', 'round', 'group_stage', 'start_date', 'index']
list_filter = [MatchTypeListFilter, MatchTournamentListFilter, SimpleIndexListFilter]
class GroupStageAdmin(admin.ModelAdmin):
class GroupStageAdmin(AutoUpdateAdmin):
list_display = ['tournament', 'index', 'start_date']
list_filter = [SimpleTournamentListFilter]
class ClubAdmin(admin.ModelAdmin):
class ClubAdmin(AutoUpdateAdmin):
list_display = ['name', 'acronym', 'city', 'creator', 'events_count', 'broadcast_code']
search_fields = ('name', 'acronym')
class PurchaseAdmin(admin.ModelAdmin):
class PurchaseAdmin(AutoUpdateAdmin):
list_display = ['user', 'identifier', 'product_id', 'quantity', 'purchase_date', 'revocation_date']
class CourtAdmin(admin.ModelAdmin):
class CourtAdmin(AutoUpdateAdmin):
list_display = ['index', 'name', 'club']
class DateIntervalAdmin(admin.ModelAdmin):
class DateIntervalAdmin(AutoUpdateAdmin):
list_display = ['court_index', 'event']
class FailedApiCallAdmin(admin.ModelAdmin):
class FailedApiCallAdmin(AutoUpdateAdmin):
list_display = ['date', 'user', 'type', 'error']
list_filter = ['user']
class LogAdmin(admin.ModelAdmin):
class LogAdmin(AutoUpdateAdmin):
list_display = ['date', 'user', 'message']
list_filter = ['user']
class DeviceTokenAdmin(admin.ModelAdmin):
class DeviceTokenAdmin(AutoUpdateAdmin):
list_display = ['user', 'value']
class ModelLogAdmin(admin.ModelAdmin):
list_display = ['user', 'date', 'operation', 'model_id', 'model_name']
list_filter = ['user']
list_display = ['get_users', 'date', 'operation', 'model_id', 'model_name']
list_filter = ['users']
ordering = ['-date']
class DataAccessAdmin(admin.ModelAdmin):
list_display = ['owner', 'get_shared_users', 'model_name', 'model_id', 'last_hierarchy_update']
@admin.display(description='Users')
def get_users(self, obj):
return ", ".join([str(item) for item in obj.users.all()])
class DataAccessAdmin(AutoUpdateAdmin):
list_display = ['owner', 'get_shared_users', 'model_name', 'model_id']
list_filter = ['owner', 'shared_with']
ordering = ['-last_hierarchy_update']
ordering = ['-granted_at']
@admin.display(description='Shared with')
def get_shared_users(self, obj):

@ -0,0 +1,23 @@
# Generated by Django 5.1 on 2024-11-26 09:48
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('tournaments', '0100_club_last_updated_by_court_last_updated_by_and_more'),
]
operations = [
migrations.RemoveField(
model_name='modellog',
name='user',
),
migrations.AddField(
model_name='modellog',
name='users',
field=models.ManyToManyField(blank=True, related_name='model_logs', to=settings.AUTH_USER_MODEL),
),
]

@ -0,0 +1,17 @@
# Generated by Django 5.1 on 2024-11-26 10:33
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('tournaments', '0101_remove_modellog_user_modellog_users'),
]
operations = [
migrations.RemoveField(
model_name='dataaccess',
name='last_hierarchy_update',
),
]

@ -14,10 +14,13 @@ class DataAccess(BaseModel):
model_name = models.CharField(max_length=50)
model_id = models.UUIDField()
granted_at = models.DateTimeField(auto_now_add=True)
last_hierarchy_update = models.DateTimeField(default=timezone.now)
# last_hierarchy_update = models.DateTimeField(default=timezone.now)
def create_access_log(self, user, operation):
"""Create a single access log for a specific user"""
def create_revoke_access_log(self):
self.create_access_log(self.shared_with.all(), 'REVOKE_ACCESS')
def create_access_log(self, users, operation):
"""Create an access log for a list of users """
model_class = apps.get_model('tournaments', self.model_name)
try:
obj = model_class.objects.get(id=self.model_id)
@ -25,19 +28,19 @@ class DataAccess(BaseModel):
if isinstance(obj, SideStoreModel):
store_id = obj.store_id
existing_log = ModelLog.objects.filter(user=user, model_id=self.model_id, operation=operation).first()
existing_log = ModelLog.objects.filter(users__in=users, model_id=self.model_id, operation=operation).first()
if existing_log:
existing_log.date = timezone.now()
existing_log.model_operation = operation
existing_log.save()
else:
ModelLog.objects.create(
user=user,
model_log = ModelLog.objects.create(
model_id=self.model_id,
model_name=self.model_name,
operation=operation,
date=timezone.now(),
store_id=store_id
)
model_log.users.set(users)
except ObjectDoesNotExist:
pass

@ -4,7 +4,7 @@ from . import ModelOperation
class ModelLog(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
user = models.ForeignKey('CustomUser', blank=True, null=True, on_delete=models.SET_NULL, related_name='model_logs')
users = models.ManyToManyField('CustomUser', related_name='model_logs', blank=True)
model_id = models.UUIDField()
operation = models.CharField(choices=ModelOperation.choices, max_length=50)
date = models.DateTimeField()

@ -1,6 +1,6 @@
import random
import string
from django.db.models.signals import post_save, pre_delete, post_delete, m2m_changed
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete, m2m_changed
from django.db.transaction import DatabaseError
from django.dispatch import receiver
from django.conf import settings
@ -8,30 +8,46 @@ from django.apps import apps
from django.utils import timezone
from django.db.models import Q
from .models import Club, FailedApiCall, CustomUser, Log, DataAccess, ModelLog, BaseModel
from .models import Club, FailedApiCall, CustomUser, Log, DataAccess, ModelLog, ModelOperation, BaseModel, SideStoreModel
import requests
from channels.layers import get_channel_layer
from asgiref.sync import async_to_sync
from threading import Timer
from functools import partial
# Synchronization
@receiver([pre_save, pre_delete])
def synchronization_prepare(sender, instance, created=False, **kwargs):
if not isinstance(instance, BaseModel):
return
if sender in [FailedApiCall, Log, ModelLog]:
return
save_model_log_if_possible(instance, kwargs.get('signal'), created)
# if not isinstance(instance, DataAccess):
# update_data_access(instance)
@receiver([post_save, post_delete])
def synchronization_notifications(sender, instance, **kwargs):
def synchronization_notifications(sender, instance, created=False, **kwargs):
"""
Signal handler that sends notifications through WebSocket channels when model instances are saved or deleted.
The function creates a WebSocket group name for each affected user and sends a sync update message
to all clients connected to that group.
"""
if not isinstance(instance, BaseModel):
return
if sender in [FailedApiCall, Log, ModelLog]:
return
if kwargs.get('signal') == post_delete:
delete_data_access_if_necessary(instance.id)
print(f'*** signals {sender}')
# print(f'*** signals {sender}')
notify_impacted_users(instance, kwargs.get('signal'))
def notify_impacted_users(instance, signal):
user_ids = set()
# add impacted users
if isinstance(instance, CustomUser):
@ -42,64 +58,126 @@ def synchronization_notifications(sender, instance, **kwargs):
user_ids.add(owner.id)
if isinstance(instance, BaseModel):
data_access_query = Q(model_id=instance.id)
if kwargs.get('signal') != post_delete:
# when deleting objects, accessing reference generates DoesNotExist exceptions
if instance._users_to_notify is not None:
user_ids.update(instance._users_to_notify)
else:
print('no users to notify')
print(f'notify: {user_ids}')
for user_id in user_ids:
send_user_message(user_id)
def save_model_log_if_possible(instance, signal, created):
user = instance.last_updated_by
if user is not None:
if signal == post_save or signal == pre_save:
if created:
operation = ModelOperation.POST
else:
operation = ModelOperation.PUT
else:
operation = ModelOperation.DELETE
model_name = instance.__class__.__name__
store_id = None
if isinstance(instance, SideStoreModel):
store_id = instance.store_id
parent_model, data_access_reference_id = instance.get_parent_reference()
if data_access_reference_id is not None:
data_access_query |= Q(model_id=data_access_reference_id)
if operation == ModelOperation.DELETE: # delete now unnecessary logs
ModelLog.objects.filter(model_id=instance.id).delete()
# look for users through data access objects
data_access_list = DataAccess.objects.filter(data_access_query)
users = {user}
data_access_list = related_data_access(instance)
for data_access in data_access_list:
user_ids.add(data_access.owner.id)
for shared_user in data_access.shared_with.all():
user_ids.add(shared_user.id)
users.update(data_access.shared_with.all())
if isinstance(instance, DataAccess):
users.add(instance.owner)
users.update(instance.shared_with.all())
for user_id in user_ids:
send_user_message(user_id)
user_ids = [user.id for user in users]
print(f'users to notify: {user_ids}')
instance._users_to_notify = user_ids # save this for the post_save signal
save_model_log(users, operation, model_name, instance.id, store_id)
else:
print('>>> Model Log could not be created because instance.last_updated_by is None')
def save_model_log(users, model_operation, model_name, model_id, store_id):
now = timezone.now()
existing_log = ModelLog.objects.filter(users__in=users, model_id=model_id, operation=model_operation).first()
if existing_log:
existing_log.date = now
existing_log.model_operation = model_operation
existing_log.save()
else:
model_log = ModelLog()
model_log.operation = model_operation
model_log.date = now
model_log.model_name = model_name
model_log.model_id = model_id
model_log.store_id = store_id
model_log.save()
model_log.users.set(users)
def related_data_access(instance):
related_instances = instance.related_instances()
related_ids = [ri.id for ri in instance.related_instances()]
related_ids.append(instance.id)
return DataAccess.objects.filter(model_id__in=related_ids)
# def update_data_access(instance):
# data_access_list = related_data_access(instance)
# for data_access in data_access_list:
# date = timezone.now() if instance.last_update is None else instance.last_update
# data_access.last_hierarchy_update = date
# data_access.save()
def delete_data_access_if_necessary(model_id):
DataAccess.objects.filter(model_id=model_id).delete()
@receiver(m2m_changed, sender=DataAccess.shared_with.through)
def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs):
users = CustomUser.objects.filter(id__in=pk_set)
if action == "post_add":
instance.create_access_log(users, 'GRANT_ACCESS')
elif action == "post_remove":
instance.create_access_log(users, 'REVOKE_ACCESS')
for user_id in pk_set:
send_user_message(user_id)
def send_user_message(user_id):
group_name = f"sync_{user_id}"
print(f">>> send to group {group_name}")
# Send to all clients in the sync group
if not hasattr(send_user_message, '_buffer'):
send_user_message._buffer = set()
send_user_message._timer = None
send_user_message._buffer.add(user_id)
if send_user_message._timer:
send_user_message._timer.cancel()
def send_buffered_messages():
channel_layer = get_channel_layer()
for buffered_id in send_user_message._buffer:
group_name = f"sync_{buffered_id}"
print(f">>> send to group {group_name}")
async_to_sync(channel_layer.group_send)(
group_name, {"type": "sync.update", "message": "hello"}
)
send_user_message._buffer.clear()
send_user_message._timer = None
@receiver(m2m_changed, sender=DataAccess.shared_with.through)
def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs):
if action == "post_add":
for user_id in pk_set:
user = CustomUser.objects.get(id=user_id)
instance.create_access_log(user, 'GRANT_ACCESS')
send_user_message(user_id)
elif action == "post_remove":
for user_id in pk_set:
user = CustomUser.objects.get(id=user_id)
instance.create_access_log(user, 'REVOKE_ACCESS')
send_user_message(user_id)
send_user_message._timer = Timer(0.1, send_buffered_messages)
send_user_message._timer.start()
@receiver(pre_delete, sender=DataAccess)
def revoke_access_after_delete(sender, instance, **kwargs):
for user in instance.shared_with.all():
instance.create_access_log(user, 'REVOKE_ACCESS')
# # Store the users in a temporary attribute that we can access after deletion
# instance._users_to_revoke = list(instance.shared_with.all())
# @receiver(post_delete, sender=DataAccess)
# def revoke_access_after_delete(sender, instance, **kwargs):
# # Create revoke logs for all previously stored users
# if hasattr(instance, '_users_to_revoke'):
# for user in instance._users_to_revoke:
# instance.create_access_log(user, 'REVOKE_ACCESS')
instance.create_revoke_access_log()
# Others

Loading…
Cancel
Save