From 9a93e2d6adc0e7aef11304d79f2192e643c1c13f Mon Sep 17 00:00:00 2001 From: Laurent Date: Thu, 12 Jun 2025 15:58:59 +0200 Subject: [PATCH 01/16] adds store_id for DataAccess --- sync/migrations/0007_dataaccess_store_id.py | 18 +++++++++ sync/models/data_access.py | 43 ++++++++------------- 2 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 sync/migrations/0007_dataaccess_store_id.py diff --git a/sync/migrations/0007_dataaccess_store_id.py b/sync/migrations/0007_dataaccess_store_id.py new file mode 100644 index 0000000..8113e21 --- /dev/null +++ b/sync/migrations/0007_dataaccess_store_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1 on 2025-06-12 13:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0006_alter_modellog_operation'), + ] + + operations = [ + migrations.AddField( + model_name='dataaccess', + name='store_id', + field=models.CharField(default='', max_length=100), + ), + ] diff --git a/sync/models/data_access.py b/sync/models/data_access.py index 5a38ae3..5f54b11 100644 --- a/sync/models/data_access.py +++ b/sync/models/data_access.py @@ -17,6 +17,7 @@ class DataAccess(BaseModel): shared_with = models.ManyToManyField(settings.AUTH_USER_MODEL, related_name='shared_data') model_name = models.CharField(max_length=50) model_id = models.UUIDField() + store_id = models.CharField(max_length=100, default="") # a value matching LeStorage directory sub-stores. Matches the name of the directory. granted_at = models.DateTimeField(auto_now_add=True) def delete_dependencies(self): @@ -35,33 +36,23 @@ class DataAccess(BaseModel): """Create an access log for a list of users """ model_class = model_registry.get_model(self.model_name) if model_class: - try: - obj = model_class.objects.get(id=self.model_id) - store_id = None - if isinstance(obj, SideStoreModel): - store_id = obj.store_id - - for user in users: + for user in users: + logger.info(f'=== create ModelLog for: {operation} > {users}') - logger.info(f'=== create ModelLog for: {operation} > {users}') - - existing_log = ModelLog.objects.filter(user=user, model_id=self.model_id, operation=operation).first() - if existing_log: - existing_log.date = timezone.now() - existing_log.model_operation = operation - existing_log.save() - else: - ModelLog.objects.create( - user=user, - model_id=self.model_id, - model_name=self.model_name, - operation=operation, - date=timezone.now(), - store_id=store_id - ) - except ObjectDoesNotExist: - logger.warn(f'!!! object does not exists any more: {self.model_name} : {self.model_id} : {operation}') - pass + existing_log = ModelLog.objects.filter(user=user, model_id=self.model_id, operation=operation).first() + if existing_log: + existing_log.date = timezone.now() + existing_log.model_operation = operation + existing_log.save() + else: + ModelLog.objects.create( + user=user, + model_id=self.model_id, + model_name=self.model_name, + operation=operation, + date=timezone.now(), + store_id=self.store_id + ) else: logger.warn(f'!!!model not found: {self.model_name}') From f82692f13e43ddf80bd0576af92b367a2dbc4c08 Mon Sep 17 00:00:00 2001 From: Laurent Date: Thu, 12 Jun 2025 16:08:03 +0200 Subject: [PATCH 02/16] logs more --- sync/signals.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sync/signals.py b/sync/signals.py index e0c0fba..211b416 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -41,8 +41,7 @@ def presave_handler(sender, instance, **kwargs): sig_type = 'pre_save' else: sig_type = 'pre_delete' - logger.info(f'* {sig_type} : {instance.__class__.__name__} > impacted users = {users}') - + # logger.info(f'* {sig_type} : {instance.__class__.__name__} > impacted users = {users}') @receiver([post_save, post_delete]) def synchronization_notifications(sender, instance, created=False, **kwargs): @@ -139,7 +138,7 @@ def save_model_log(users, model_operation, model_name, model_id, store_id): with transaction.atomic(): created_logs = [] for user in users: - logger.debug(f'Creating ModelLog for user {user.id} ({user.username})') + logger.info(f'Creating ModelLog for user {user.id} ({user.username})') model_log = ModelLog( user=user, operation=model_operation, @@ -150,7 +149,7 @@ def save_model_log(users, model_operation, model_name, model_id, store_id): ) model_log.save() created_logs.append(model_log.id) - logger.debug(f'Successfully created ModelLog {model_log.id}') + logger.info(f'Successfully created ModelLog {model_log.id}') logger.info(f'*** Successfully created {len(created_logs)} ModelLogs: {created_logs}') @@ -159,7 +158,7 @@ def save_model_log(users, model_operation, model_name, model_id, store_id): if persisted_count != len(created_logs): logger.error(f'*** PERSISTENCE VERIFICATION FAILED! Created {len(created_logs)} ModelLogs but only {persisted_count} were persisted to database') else: - logger.debug(f'*** PERSISTENCE VERIFIED: All {persisted_count} ModelLogs successfully persisted') + logger.info(f'*** PERSISTENCE VERIFIED: All {persisted_count} ModelLogs successfully persisted') except Exception as e: logger.error(f'*** FAILED to create ModelLogs for: {model_operation} {model_name}, users: {[u.id for u in users]}, error: {e}', exc_info=True) From bf8f103bdad29fc99c76fd2d040022456f815d48 Mon Sep 17 00:00:00 2001 From: Laurent Date: Thu, 12 Jun 2025 16:12:04 +0200 Subject: [PATCH 03/16] simplify log creation --- sync/models/data_access.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/sync/models/data_access.py b/sync/models/data_access.py index 5f54b11..e0455ec 100644 --- a/sync/models/data_access.py +++ b/sync/models/data_access.py @@ -38,21 +38,29 @@ class DataAccess(BaseModel): if model_class: for user in users: logger.info(f'=== create ModelLog for: {operation} > {users}') + ModelLog.objects.create( + user=user, + model_id=self.model_id, + model_name=self.model_name, + operation=operation, + date=timezone.now(), + store_id=self.store_id + ) - existing_log = ModelLog.objects.filter(user=user, model_id=self.model_id, operation=operation).first() - if existing_log: - existing_log.date = timezone.now() - existing_log.model_operation = operation - existing_log.save() - else: - ModelLog.objects.create( - user=user, - model_id=self.model_id, - model_name=self.model_name, - operation=operation, - date=timezone.now(), - store_id=self.store_id - ) + # existing_log = ModelLog.objects.filter(user=user, model_id=self.model_id, operation=operation).first() + # if existing_log: + # existing_log.date = timezone.now() + # existing_log.model_operation = operation + # existing_log.save() + # else: + # ModelLog.objects.create( + # user=user, + # model_id=self.model_id, + # model_name=self.model_name, + # operation=operation, + # date=timezone.now(), + # store_id=self.store_id + # ) else: logger.warn(f'!!!model not found: {self.model_name}') From d97888c887a5e48073497dcd09cbc2a156f6c9d4 Mon Sep 17 00:00:00 2001 From: Laurent Date: Fri, 13 Jun 2025 10:45:24 +0200 Subject: [PATCH 04/16] put DataAccess auto-delete in the pre_delete BaseModel signal --- sync/models/data_access.py | 2 +- sync/signals.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sync/models/data_access.py b/sync/models/data_access.py index e0455ec..23425b3 100644 --- a/sync/models/data_access.py +++ b/sync/models/data_access.py @@ -7,7 +7,7 @@ from django.conf import settings from ..registry import model_registry import uuid -from . import ModelLog, SideStoreModel, BaseModel +from . import ModelLog, BaseModel import logging logger = logging.getLogger(__name__) diff --git a/sync/signals.py b/sync/signals.py index 211b416..76cc7c5 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -39,7 +39,9 @@ def presave_handler(sender, instance, **kwargs): if signal == pre_save: detect_foreign_key_changes_for_shared_instances(sender, instance) sig_type = 'pre_save' - else: + elif signal == pre_delete: + if hasattr(instance, 'id'): + DataAccess.objects.filter(model_id=instance.id).delete() sig_type = 'pre_delete' # logger.info(f'* {sig_type} : {instance.__class__.__name__} > impacted users = {users}') @@ -274,12 +276,12 @@ def process_foreign_key_changes(sender, instance, **kwargs): ### Data Access -@receiver(post_delete) -def delete_data_access_if_necessary(sender, instance, **kwargs): - if not isinstance(instance, BaseModel): - return - if hasattr(instance, 'id'): - DataAccess.objects.filter(model_id=instance.id).delete() +# @receiver(post_delete) +# def delete_data_access_if_necessary(sender, instance, **kwargs): +# if not isinstance(instance, BaseModel): +# return +# if hasattr(instance, 'id'): +# DataAccess.objects.filter(model_id=instance.id).delete() @receiver(m2m_changed, sender=DataAccess.shared_with.through) def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs): From 9b739c85a2d88f4547e57cc45b51c08c19685203 Mon Sep 17 00:00:00 2001 From: Laurent Date: Fri, 13 Jun 2025 11:03:16 +0200 Subject: [PATCH 05/16] remove logs --- sync/signals.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sync/signals.py b/sync/signals.py index 76cc7c5..b49371a 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -41,7 +41,9 @@ def presave_handler(sender, instance, **kwargs): sig_type = 'pre_save' elif signal == pre_delete: if hasattr(instance, 'id'): - DataAccess.objects.filter(model_id=instance.id).delete() + data_access_list = DataAccess.objects.filter(model_id=instance.id) + logger.info(f'>>> delete {data_access_list.count()} DataAccess') + data_access_list.delete() sig_type = 'pre_delete' # logger.info(f'* {sig_type} : {instance.__class__.__name__} > impacted users = {users}') @@ -140,7 +142,7 @@ def save_model_log(users, model_operation, model_name, model_id, store_id): with transaction.atomic(): created_logs = [] for user in users: - logger.info(f'Creating ModelLog for user {user.id} ({user.username})') + # logger.info(f'Creating ModelLog for user {user.id} ({user.username})') model_log = ModelLog( user=user, operation=model_operation, @@ -151,9 +153,9 @@ def save_model_log(users, model_operation, model_name, model_id, store_id): ) model_log.save() created_logs.append(model_log.id) - logger.info(f'Successfully created ModelLog {model_log.id}') + # logger.info(f'Successfully created ModelLog {model_log.id}') - logger.info(f'*** Successfully created {len(created_logs)} ModelLogs: {created_logs}') + # logger.info(f'*** Successfully created {len(created_logs)} ModelLogs: {created_logs}') # Verify ModelLogs were actually persisted persisted_count = ModelLog.objects.filter(id__in=created_logs).count() From fc21dc2b936ca963929ecd14c55a8cd2f7a52470 Mon Sep 17 00:00:00 2001 From: Laurent Date: Fri, 13 Jun 2025 17:35:21 +0200 Subject: [PATCH 06/16] improve perf + log --- sync/models/data_access.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/sync/models/data_access.py b/sync/models/data_access.py index 23425b3..a371eb7 100644 --- a/sync/models/data_access.py +++ b/sync/models/data_access.py @@ -1,8 +1,8 @@ from django.db import models from django.utils import timezone -# from django.apps import apps from django.core.exceptions import ObjectDoesNotExist from django.conf import settings +from django.db import transaction from ..registry import model_registry import uuid @@ -71,10 +71,13 @@ class DataAccess(BaseModel): obj = model_class.objects.get(id=self.model_id) related_instance = obj.related_instances() related_instance.append(obj) - for instance in related_instance: - if isinstance(instance, BaseModel): - instance.add_data_access_relation(self) - instance.save() + + with transaction.atomic(): + for instance in related_instance: + logger.info(f'adds DataAccess to {instance.__class__.__name__}') + if isinstance(instance, BaseModel): + instance.add_data_access_relation(self) + instance.save() except ObjectDoesNotExist: pass @@ -85,9 +88,11 @@ class DataAccess(BaseModel): obj = model_class.objects.get(id=self.model_id) related_instance = obj.related_instances() related_instance.append(obj) - for instance in related_instance: - if isinstance(instance, BaseModel): - instance.remove_data_access_relation(self) - instance.save() + + with transaction.atomic(): + for instance in related_instance: + if isinstance(instance, BaseModel): + instance.remove_data_access_relation(self) + instance.save() except ObjectDoesNotExist: pass From 90e7f4216ec5f425bf1662cdd7ebf41f57e08e75 Mon Sep 17 00:00:00 2001 From: laurent Date: Fri, 20 Jun 2025 15:36:46 +0200 Subject: [PATCH 07/16] improvements --- sync/model_manager.py | 0 sync/models/base.py | 89 ++++++++++++--- sync/models/data_access.py | 3 +- sync/registry.py | 180 ++++++++++++++++++++++++++++- sync/signals.py | 189 +++++++++++++++++-------------- sync/utils.py | 1 + sync/views.py | 81 ++++++++++++- tournaments/models/match.py | 24 ++-- tournaments/models/team_score.py | 32 ++++-- 9 files changed, 470 insertions(+), 129 deletions(-) create mode 100644 sync/model_manager.py diff --git a/sync/model_manager.py b/sync/model_manager.py new file mode 100644 index 0000000..e69de29 diff --git a/sync/models/base.py b/sync/models/base.py index 8668e9d..dc123a7 100644 --- a/sync/models/base.py +++ b/sync/models/base.py @@ -1,9 +1,14 @@ from django.db import models from django.utils.timezone import now from django.conf import settings +from django.apps import apps + from typing import List, Set -from django.apps import apps +from collections import defaultdict +import logging + +logger = logging.getLogger(__name__) class BaseModel(models.Model): creation_date = models.DateTimeField(default=now, editable=False) @@ -28,22 +33,22 @@ class BaseModel(models.Model): else: return None + def data_identifier_dict(self): + return { + 'model_id': self.id, + 'store_id': None + } + def update_data_access_list(self): - related_instances = self.related_instances() - related_ids = [ri.id for ri in related_instances] - related_ids.append(self.id) - - DataAccess = apps.get_model('sync', 'DataAccess') - data_accesses = DataAccess.objects.filter(model_id__in=related_ids) - for data_access in data_accesses: - self.add_data_access_relation(data_access) - - # add data_access to children who might not had the relationship - # if data_accesses: - # for child in self.get_children_by_model(): - # if len(child.data_access_ids) == 0: - # for data_access in data_accesses: - # self.add_data_access_relation(data_access) + related_instances = self.sharing_related_instances() + data_access_ids = {instance.data_access_ids for instance in related_instances} + data_access_ids.update(self.data_access_ids) + self.data_access_ids = data_access_ids + + # DataAccess = apps.get_model('sync', 'DataAccess') + # data_accesses = DataAccess.objects.filter(model_id__in=related_ids) + # for data_access in data_accesses: + # self.add_data_access_relation(data_access) def add_data_access_relation(self, data_access): str_id = str(data_access.id) @@ -187,8 +192,60 @@ class BaseModel(models.Model): return None + def sharing_related_instances(self): + """ + Get all related instances (both children and parents) recursively + """ + instances = [] + processed_objects = set() + instances.extend(self.get_shared_children(processed_objects)) + + processed_objects = set() + instances.extend(self.get_recursive_parents(processed_objects)) + + return instances + + def get_shared_children(self, processed_objects): + + sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) + relationships = sync_models[self.__class__.__name__] + if relationships: + return self.get_shared_children_from_relationships(relationships, processed_objects) + else: + return self.get_recursive_children(processed_objects) + + def get_shared_children_from_relationships(self, relationships, processed_objects): + + # print(f'relationships = {relationships}') + current = [self] + for relationship in relationships: + # print(f'> relationship = {relationship}') + values = [] + for item in current: + value = getattr(item, relationship) + + if hasattr(value, 'all') and callable(value.all): + # This is a queryset from a reverse relationship + for related_obj in value.all(): + processed_objects.add(related_obj) + values.extend(value.all()) + else: + processed_objects.add(value) + + values.append(value) + current = values + + logger.info(f'+++ shared children = {processed_objects}') + return processed_objects + class SideStoreModel(BaseModel): store_id = models.CharField(max_length=100, default="") # a value matching LeStorage directory sub-stores. Matches the name of the directory. class Meta: abstract = True + + def data_identifier_dict(self): + return { + 'model_id': self.id, + 'store_id': self.store_id + } diff --git a/sync/models/data_access.py b/sync/models/data_access.py index a371eb7..1e41874 100644 --- a/sync/models/data_access.py +++ b/sync/models/data_access.py @@ -69,7 +69,8 @@ class DataAccess(BaseModel): if model_class: try: obj = model_class.objects.get(id=self.model_id) - related_instance = obj.related_instances() + + related_instance = obj.sharing_related_instances() related_instance.append(obj) with transaction.atomic(): diff --git a/sync/registry.py b/sync/registry.py index f27b2e2..1f42797 100644 --- a/sync/registry.py +++ b/sync/registry.py @@ -1,8 +1,16 @@ from django.conf import settings from django.apps import apps -from .models import BaseModel from django.contrib.auth import get_user_model + +from .models import BaseModel + import threading +import logging + +from typing import List, Optional, Dict + + +logger = logging.getLogger(__name__) User = get_user_model() @@ -105,3 +113,173 @@ class RelatedUsersRegistry: # Global instance related_users_registry = RelatedUsersRegistry() + + +class SyncModelChildrenManager: + """ + Manager class for handling model children sharing configuration. + Reads the SYNC_MODEL_CHILDREN_SHARING setting once and builds a bidirectional + relationship graph for efficient lookup. + """ + + def __init__(self): + """Initialize the manager by reading the Django setting and building the relationship graph.""" + self._model_relationships = getattr( + settings, + 'SYNC_MODEL_CHILDREN_SHARING', + {} + ) + self._relationship_graph = self._build_relationship_graph() + logger.info(f'self._relationship_graph = {self._relationship_graph}') + + def _build_relationship_graph(self) -> Dict[str, List[List[str]]]: + """ + Build a bidirectional relationship graph. + + Returns: + Dict[str, List[List[str]]]: Dictionary where keys are model names and values + are lists of relationship paths (arrays of relationship names). + """ + graph = {} + + # Add direct relationships (original models to their children) + for model_name, relationships in self._model_relationships.items(): + if model_name not in graph: + graph[model_name] = [] + + # Add direct relationships as single-item arrays + for relationship in relationships: + graph[model_name].append([relationship]) + + # Build reverse relationships (children back to original models) + for original_model_name, relationships in self._model_relationships.items(): + try: + original_model = model_registry.get_model(original_model_name) + if original_model is None: + continue + + for relationship_name in relationships: + # Get the related model through _meta + try: + field = None + # Try to find the field in the model's _meta + for f in original_model._meta.get_fields(): + if hasattr(f, 'related_name') and f.related_name == relationship_name: + field = f + break + elif hasattr(f, 'name') and f.name == relationship_name: + field = f + break + + if field is None: + continue + + # Get the related model + if hasattr(field, 'related_model'): + related_model = field.related_model + elif hasattr(field, 'model'): + related_model = field.model + else: + continue + + related_model_name = related_model.__name__ + + # Find the reverse relationship name + reverse_relationship_name = self._find_reverse_relationship( + related_model, original_model, relationship_name + ) + + if reverse_relationship_name: + # Add the reverse path + if related_model_name not in graph: + graph[related_model_name] = [] + + # The path back is just the reverse relationship name + graph[related_model_name].append([reverse_relationship_name]) + + except Exception as e: + # Skip problematic relationships + continue + + except Exception as e: + # Skip problematic models + continue + + return graph + + def _find_reverse_relationship(self, from_model, to_model, original_relationship_name): + """ + Find the reverse relationship name from from_model to to_model. + + Args: + from_model: The model to search relationships from + to_model: The target model to find relationship to + original_relationship_name: The original relationship name for context + + Returns: + str or None: The reverse relationship name if found + """ + try: + for field in from_model._meta.get_fields(): + # Check ForeignKey, OneToOneField fields + if hasattr(field, 'related_model') and field.related_model == to_model: + # Check if this field has a related_name that matches our original relationship + if hasattr(field, 'related_name') and field.related_name == original_relationship_name: + # This is the reverse of our original relationship + return field.name + elif not hasattr(field, 'related_name') or field.related_name is None: + # Default reverse relationship name + default_name = f"{to_model._meta.model_name}" + if default_name == original_relationship_name.rstrip('s'): # Simple heuristic + return field.name + + # Check reverse relationships + if hasattr(field, 'field') and hasattr(field.field, 'model'): + if field.field.model == to_model: + if field.get_accessor_name() == original_relationship_name: + return field.field.name + + except Exception: + pass + + return None + + def get_relationships(self, model_name: str) -> List[str]: + """ + Get the list of direct relationships for a given model name. + + Args: + model_name (str): The name of the model to look up + + Returns: + List[str]: List of relationship names for the model. + Returns empty list if model is not found. + """ + return self._model_relationships.get(model_name, []) + + def get_relationship_paths(self, model_name: str) -> List[List[str]]: + """ + Get all relationship paths for a given model name. + This includes both direct relationships and reverse paths. + + Args: + model_name (str): The name of the model to look up + + Returns: + List[List[str]]: List of relationship paths (each path is a list of relationship names). + Returns empty list if model is not found. + """ + return self._relationship_graph.get(model_name, []) + + def get_relationship_graph(self) -> Dict[str, List[List[str]]]: + """ + Get the complete relationship graph. + + Returns: + Dict[str, List[List[str]]]: The complete relationship graph + """ + return self._relationship_graph.copy() + + +# Create a singleton instance to use throughout the application +sync_model_manager = SyncModelChildrenManager() diff --git a/sync/signals.py b/sync/signals.py index b49371a..7e275a8 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -11,6 +11,8 @@ from .ws_sender import websocket_sender from .registry import device_registry, related_users_registry import logging +import sys +import traceback logger = logging.getLogger(__name__) @@ -21,31 +23,41 @@ User = get_user_model() @receiver([pre_save, pre_delete]) def presave_handler(sender, instance, **kwargs): - # some other classes are excluded in settings_app.py: SYNC_APPS - if not isinstance(instance, (BaseModel, User)): - return - - signal = kwargs.get('signal') - # avoid crash in manage.py createsuperuser + delete user in the admin - if isinstance(instance, User) and (instance._state.db is None or signal == pre_delete): - return + try: + # some other classes are excluded in settings_app.py: SYNC_APPS + if not isinstance(instance, (BaseModel, User)) or isinstance(instance, DataAccess): + return - users = related_users(instance) + signal = kwargs.get('signal') + # avoid crash in manage.py createsuperuser + delete user in the admin + if isinstance(instance, User) and (instance._state.db is None or signal == pre_delete): + return - related_users_registry.register(instance.id, users) - # user_ids = [user.id for user in users] + users = related_users(instance) + related_users_registry.register(instance.id, users) + + if signal == pre_save: + detect_foreign_key_changes_for_shared_instances(sender, instance) + sig_type = 'pre_save' + elif signal == pre_delete: + # if hasattr(instance, 'id'): + # try: + # data_access_list = DataAccess.objects.filter(model_id=instance.id) + # if data_access_list: + # logger.info(f'>>> {instance.__class__.__name__} {instance.id} : delete {data_access_list.count()} DataAccess') + # data_access_list.delete() + # except Exception as e: + # logger.info(f'*** ERRRRRRR: {e}') + # logger.info(traceback.format_exc()) + # raise + + sig_type = 'pre_delete' + # logger.info(f'* {sig_type} : {instance.__class__.__name__} > impacted users = {users}') - if signal == pre_save: - detect_foreign_key_changes_for_shared_instances(sender, instance) - sig_type = 'pre_save' - elif signal == pre_delete: - if hasattr(instance, 'id'): - data_access_list = DataAccess.objects.filter(model_id=instance.id) - logger.info(f'>>> delete {data_access_list.count()} DataAccess') - data_access_list.delete() - sig_type = 'pre_delete' - # logger.info(f'* {sig_type} : {instance.__class__.__name__} > impacted users = {users}') + except Exception as e: + logger.info(f'*** ERROR: {e}') + raise @receiver([post_save, post_delete]) def synchronization_notifications(sender, instance, created=False, **kwargs): @@ -59,47 +71,31 @@ def synchronization_notifications(sender, instance, created=False, **kwargs): if not isinstance(instance, BaseModel) and not isinstance(instance, User): return - process_foreign_key_changes(sender, instance, **kwargs) - - signal = kwargs.get('signal') - save_model_log_if_possible(instance, signal, created) - notify_impacted_users(instance) - - related_users_registry.unregister(instance.id) + try: + process_foreign_key_changes(sender, instance, **kwargs) + signal = kwargs.get('signal') + save_model_log_if_possible(instance, signal, created) + notify_impacted_users(instance) + related_users_registry.unregister(instance.id) + except Exception as e: + logger.info(f'*** ERROR2: {e}') + logger.info(traceback.format_exc()) + raise def notify_impacted_users(instance): - # print(f'*** notify_impacted_users for instance: {instance}') - # user_ids = set() - # # add impacted users - # if isinstance(instance, User): - # user_ids.add(instance.id) - # elif isinstance(instance, BaseModel): - # owner = instance.last_updated_by - # if owner: - # user_ids.add(owner.id) - - # if isinstance(instance, BaseModel): - # if hasattr(instance, '_users_to_notify'): - # user_ids.update(instance._users_to_notify) - # else: - # print('no users to notify') - device_id = device_registry.get_device_id(instance.id) users = related_users_registry.get_users(instance.id) if users: user_ids = [user.id for user in users] websocket_sender.send_message(user_ids, device_id) - # print(f'notify device: {device_id}, users = {user_ids}') - # for user_id in user_ids: - # websocket_sender.send_user_message(user_id, device_id) device_registry.unregister(instance.id) def save_model_log_if_possible(instance, signal, created): users = related_users_registry.get_users(instance.id) - logger.debug(f'*** save_model_log_if_possible >>> users from registry = {users}, instance = {instance}') + # logger.info(f'*** save_model_log_if_possible >>> users from registry = {users}, instance = {instance}') if not users: logger.warning(f'!!! Registry returned empty users for instance {instance.id} ({instance.__class__.__name__})') @@ -121,8 +117,8 @@ def save_model_log_if_possible(instance, signal, created): if isinstance(instance, SideStoreModel): store_id = instance.store_id - if operation == ModelOperation.DELETE: # delete now unnecessary logs - ModelLog.objects.filter(model_id=instance.id).delete() + # if operation == ModelOperation.DELETE: # delete now unnecessary logs + # ModelLog.objects.filter(model_id=instance.id).delete() # user_ids = [user.id for user in users] # # print(f'users to notify: {user_ids}') @@ -133,16 +129,14 @@ def save_model_log_if_possible(instance, signal, created): logger.info(f'!!! Model Log could not be created because no linked user could be found: {instance.__class__.__name__} {instance}, {signal}') def save_model_log(users, model_operation, model_name, model_id, store_id): - device_id = device_registry.get_device_id(model_id) - logger.info(f'*** creating ModelLogs for: {model_operation} {model_name} : {users}') try: with transaction.atomic(): created_logs = [] for user in users: - # logger.info(f'Creating ModelLog for user {user.id} ({user.username})') + # logger.info(f'Creating ModelLog for user {user.id} - user exists: {User.objects.filter(id=user.id).exists()}') model_log = ModelLog( user=user, operation=model_operation, @@ -152,20 +146,19 @@ def save_model_log(users, model_operation, model_name, model_id, store_id): device_id=device_id ) model_log.save() + # logger.info(f'ModelLog saved with ID: {model_log.id}') created_logs.append(model_log.id) - # logger.info(f'Successfully created ModelLog {model_log.id}') - # logger.info(f'*** Successfully created {len(created_logs)} ModelLogs: {created_logs}') + # Immediate verification within transaction + immediate_count = ModelLog.objects.filter(id__in=created_logs).count() + # logger.info(f'*** Within transaction: Created {len(created_logs)}, found {immediate_count}') - # Verify ModelLogs were actually persisted - persisted_count = ModelLog.objects.filter(id__in=created_logs).count() - if persisted_count != len(created_logs): - logger.error(f'*** PERSISTENCE VERIFICATION FAILED! Created {len(created_logs)} ModelLogs but only {persisted_count} were persisted to database') - else: - logger.info(f'*** PERSISTENCE VERIFIED: All {persisted_count} ModelLogs successfully persisted') + # Verification after transaction commits + persisted_count = ModelLog.objects.filter(id__in=created_logs).count() + # logger.info(f'*** After transaction: Created {len(created_logs)}, persisted {persisted_count}') except Exception as e: - logger.error(f'*** FAILED to create ModelLogs for: {model_operation} {model_name}, users: {[u.id for u in users]}, error: {e}', exc_info=True) + logger.error(f'*** Exception during ModelLog creation: {e}', exc_info=True) raise # with transaction.atomic(): @@ -209,7 +202,6 @@ def detect_foreign_key_changes_for_shared_instances(sender, instance): return data_access_list = related_data_access(instance) - # print(f'FK change > DA count = {len(data_access_list)}') if data_access_list: try: old_instance = sender.objects.get(pk=instance.pk) @@ -278,12 +270,12 @@ def process_foreign_key_changes(sender, instance, **kwargs): ### Data Access -# @receiver(post_delete) -# def delete_data_access_if_necessary(sender, instance, **kwargs): -# if not isinstance(instance, BaseModel): -# return -# if hasattr(instance, 'id'): -# DataAccess.objects.filter(model_id=instance.id).delete() +@receiver(post_delete) +def delete_data_access_if_necessary(sender, instance, **kwargs): + if not isinstance(instance, BaseModel): + return + if hasattr(instance, 'id'): + DataAccess.objects.filter(model_id=instance.id).delete() @receiver(m2m_changed, sender=DataAccess.shared_with.through) def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs): @@ -309,26 +301,41 @@ def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs): @receiver(post_save, sender=DataAccess) def data_access_post_save(sender, instance, **kwargs): - instance.add_references() # create DataAccess references on hierarchy + try: + instance.add_references() # create DataAccess references on hierarchy - if instance.related_user: - evaluate_if_user_should_sync(instance.related_user) + if instance.related_user: + evaluate_if_user_should_sync(instance.related_user) + except Exception as e: + logger.info(f'*** ERROR3: {e}') + logger.info(traceback.format_exc()) + raise @receiver(pre_delete, sender=DataAccess) def revoke_access_after_delete(sender, instance, **kwargs): - instance.cleanup_references() - instance.create_revoke_access_log() - related_users_registry.register(instance.id, instance.shared_with.all()) + try: + instance.cleanup_references() + instance.create_revoke_access_log() + related_users_registry.register(instance.id, instance.shared_with.all()) - instance._user = instance.related_user + instance._user = instance.related_user + except Exception as e: + logger.info(f'*** ERROR4: {e}') + logger.info(traceback.format_exc()) + raise @receiver(post_delete, sender=DataAccess) def data_access_post_delete(sender, instance, **kwargs): - notify_impacted_users(instance) + try: + notify_impacted_users(instance) - if not hasattr(instance, '_user') or not instance._user: - return - evaluate_if_user_should_sync(instance._user) + if not hasattr(instance, '_user') or not instance._user: + return + evaluate_if_user_should_sync(instance._user) + except Exception as e: + logger.info(f'*** ERROR5: {e}') + logger.info(traceback.format_exc()) + raise def related_users(instance): users = set() @@ -373,9 +380,14 @@ def evaluate_if_user_should_sync(user): @receiver(post_save, sender=Device) def device_created(sender, instance, **kwargs): - if not instance.user: - return - evaluate_if_user_should_sync(instance.user) + try: + if not instance.user: + return + evaluate_if_user_should_sync(instance.user) + except Exception as e: + logger.info(f'*** ERROR6: {e}') + logger.info(traceback.format_exc()) + raise @receiver(pre_delete, sender=Device) def device_pre_delete(sender, instance, **kwargs): @@ -383,6 +395,11 @@ def device_pre_delete(sender, instance, **kwargs): @receiver(post_delete, sender=Device) def device_post_delete(sender, instance, **kwargs): - if not hasattr(instance, '_user') or not instance._user: - return - evaluate_if_user_should_sync(instance._user) + try: + if not hasattr(instance, '_user') or not instance._user: + return + evaluate_if_user_should_sync(instance._user) + except Exception as e: + logger.info(f'*** ERROR7: {e}') + logger.info(traceback.format_exc()) + raise diff --git a/sync/utils.py b/sync/utils.py index ee7ff80..07168bf 100644 --- a/sync/utils.py +++ b/sync/utils.py @@ -50,6 +50,7 @@ class HierarchyOrganizer: def __init__(self): self.levels = [] # List of dictionaries, each representing a level self.item_levels = {} # Keep track of items and their levels: (model_name, id) -> level + self.sharing_related_instances = {} # Keep track of items and their levels: (model_name, id) -> level def add_item(self, model_name, item_data, level): """ diff --git a/sync/views.py b/sync/views.py index 3f27041..c4dd8e3 100644 --- a/sync/views.py +++ b/sync/views.py @@ -24,6 +24,10 @@ from .models import ModelLog, BaseModel, SideStoreModel, DataAccess from .registry import model_registry, device_registry from .ws_sender import websocket_sender +import logging + +logger = logging.getLogger(__name__) + # class HierarchyApiView(APIView): def add_children_hierarchy(instance, models_dict): @@ -358,21 +362,32 @@ class LogProcessingResult: revocations = defaultdict(list) revocations_parents_organizer = HierarchyOrganizer() - # print(f'*** process_revocations: {len(self.revoke_info)}') + # logger.info(f'$$$ process_revocations: {len(self.revoke_info)}') + sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) # First, collect all revocations for model_name, items in self.revoke_info.items(): revocations[model_name].extend(items) - # print(f'*** process_revocations for {model_name}') + logger.info(f'$$$ process_revocations for {model_name}, items = {len(items)}') # Process parent hierarchies for each revoked item model = model_registry.get_model(model_name) for item in items: + + logger.info(f'$$$ item revoked = {item}') + try: instance = model.objects.get(id=item['model_id']) - # print(f'*** process revoked item parents of {model_name} : {item['model_id']}') + logger.info(f'$$$ process revoked item parents of {model_name} : {item['model_id']}') add_parents_with_hierarchy_organizer(instance, revocations_parents_organizer) + + # if instance.__class__.__name__ in sync_models: + # sharing_related_instances = sharing_related_instances(instance, True) + # logger.info(f'$$$ get shared instances: {len(sharing_related_instances)}') + # revocations = merge_dicts_dedup(revocations, sharing_related_instances) + # # revocations_parents_organizer.sharing_related_instances = instance.sharing_related_instances() + except model.DoesNotExist: pass @@ -387,6 +402,8 @@ class LogProcessingResult: # print(f'self.shared_relationship_sets = {self.shared_relationship_sets}') # print(f'self.shared_relationship_removals = {self.shared_relationship_removals}') + logger.info('--------------------- SYNC') + return { "updates": dict(self.updates), "deletions": dict(self.deletions), @@ -451,3 +468,61 @@ class DataAccessViewSet(viewsets.ModelViewSet): if self.request.user: return self.queryset.filter(Q(related_user=self.request.user) | Q(shared_with__in=[self.request.user])) return [] + +def merge_dicts_dedup(dict1, dict2): + """Merge two dictionaries, combining arrays and removing duplicates""" + all_keys = set(dict1.keys()) | set(dict2.keys()) + + merged = {} + for key in all_keys: + arr1 = dict1.get(key, []) + arr2 = dict2.get(key, []) + # Convert to sets, union them, then back to list to remove duplicates + merged[key] = list(set(arr1) | set(arr2)) + + return merged + +def sharing_related_instances(instance, identifiers_only): + + sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) + # if self.__class__.__name__ in sync_models: + relationships = sync_models[instance.__class__.__name__] + # 'Match': {'team_scores', 'team_registration', 'player_registrations'} + + models_dict = defaultdict(dict) + + print(f'relationships = {relationships}') + current = [instance] + for relationship in relationships: + print(f'> relationship = {relationship}') + values = [] + for item in current: + value = getattr(item, relationship) + + if hasattr(value, 'all') and callable(value.all): + # This is a queryset from a reverse relationship + for related_obj in value.all(): + child_model_name = related_obj.__class__.__name__ + if identifiers_only: + models_dict[child_model_name].append(related_obj.data_identifier_dict()) + else: + serializer = get_serializer(related_obj, child_model_name) + models_dict[child_model_name].append(serializer.data) + # print(f'>>> 1 Added child for {relationship}: {child_model_name}') + values.extend(value.all()) + else: + # This is a single object + child_model_name = value.__class__.__name__ + if identifiers_only: + models_dict[child_model_name].append(value.data_identifier_dict()) + else: + serializer = get_serializer(value, child_model_name) + models_dict[child_model_name].append(serializer.data) + + # serializer = get_serializer(value, child_model_name) + # models_dict[child_model_name][value.id] = serializer.data + # print(f'>>> 2 Added child for {relationship}: {child_model_name}') + + values.append(value) + current = values + return models_dict diff --git a/tournaments/models/match.py b/tournaments/models/match.py index 25f0a15..af8660c 100644 --- a/tournaments/models/match.py +++ b/tournaments/models/match.py @@ -1,7 +1,9 @@ from django.db import models -# from tournaments.models import group_stage -from . import TournamentSubModel, Round, GroupStage, FederalMatchCategory from django.utils import timezone, formats +from django.core.exceptions import ObjectDoesNotExist + +from . import TournamentSubModel, Round, GroupStage, FederalMatchCategory + from datetime import datetime, timedelta import uuid @@ -90,14 +92,16 @@ class Match(TournamentSubModel): return '--' def stage_name(self): - if self.name: - return self.name - elif self.round: - return self.round.name() - elif self.group_stage: - return self.group_stage.display_name() - else: - return '--' + try: + if self.name: + return self.name + elif self.round: + return self.round.name() + elif self.group_stage: + return self.group_stage.display_name() + except ObjectDoesNotExist: + pass + return "--" def get_previous_round(self): # Calculate the next index diff --git a/tournaments/models/team_score.py b/tournaments/models/team_score.py index 14765a6..b4845ff 100644 --- a/tournaments/models/team_score.py +++ b/tournaments/models/team_score.py @@ -1,4 +1,6 @@ from django.db import models +from django.core.exceptions import ObjectDoesNotExist + from . import TournamentSubModel, Match, TeamRegistration, FederalMatchCategory import uuid from .match import Team # Import Team only when needed @@ -15,10 +17,14 @@ class TeamScore(TournamentSubModel): pass def __str__(self): - if self.match: - return f"{self.match.stage_name()} #{self.match.index}: {self.player_names()}" - else: - return "Empty" + try: + if self.match and self.team_registration: + return f"{self.match.stage_name()} #{self.match.index}: {self.player_names()}" + if self.match: + return f"{self.match.stage_name()} #{self.match.index}" + except ObjectDoesNotExist: + pass + return "--" def get_tournament(self): # mandatory method for TournamentSubModel if self.team_registration: @@ -48,14 +54,16 @@ class TeamScore(TournamentSubModel): # return None def player_names(self): - if self.team_registration: - if self.team_registration.name: - return self.team_registration.name - else: - names = self.team_registration.team_names() - return " - ".join(names) - else: - return "--" + try: + if self.team_registration: # this can cause an exception when deleted + if self.team_registration.name: + return self.team_registration.name + else: + names = self.team_registration.team_names() + return " - ".join(names) + except TeamRegistration.DoesNotExist: + pass + return "--" def shortened_team_names(self, forced=False): names = [] From e8d92d1216d27d647267a87a5e3c8200fe145eca Mon Sep 17 00:00:00 2001 From: Laurent Date: Sun, 22 Jun 2025 10:49:33 +0200 Subject: [PATCH 08/16] add mecasnism to get shared instance reverse paths --- sync/model_manager.py | 194 ++++++++++++++++++++++++++++++++++++ sync/models/base.py | 23 +++-- sync/models/data_access.py | 3 +- sync/registry.py | 185 ++-------------------------------- tournaments/models/event.py | 2 +- 5 files changed, 216 insertions(+), 191 deletions(-) diff --git a/sync/model_manager.py b/sync/model_manager.py index e69de29..6fdff40 100644 --- a/sync/model_manager.py +++ b/sync/model_manager.py @@ -0,0 +1,194 @@ +from django.conf import settings + +from typing import List, Dict +from .registry import model_registry + +import logging + +logger = logging.getLogger(__name__) + +class SyncModelChildrenManager: + """ + Manager class for handling model children sharing configuration. + Reads the SYNC_MODEL_CHILDREN_SHARING setting once and builds a bidirectional + relationship graph for efficient lookup. + """ + + def __init__(self): + """Initialize the manager by reading the Django setting and building the relationship graph.""" + self._model_relationships = getattr( + settings, + 'SYNC_MODEL_CHILDREN_SHARING', + {} + ) + self._relationship_graph = {} + + def _build_relationship_graph(self) -> Dict[str, List[List[str]]]: + """ + Build a bidirectional relationship graph. + + Returns: + Dict[str, List[List[str]]]: Dictionary where keys are model names and values + are lists of relationship paths (arrays of relationship names). + """ + graph = {} + + # Add direct relationships (original models to their children) + for model_name, relationships in self._model_relationships.items(): + if model_name not in graph: + graph[model_name] = [] + graph[model_name].append(relationships) + + # Build reverse relationships (children back to original models) + for original_model_name, relationships in self._model_relationships.items(): + + try: + current_model = model_registry.get_model(original_model_name) + if current_model is None: + print(f'missing {original_model_name}') + continue + + current_reverse_path = [] + + for relationship_name in relationships: + # Get the related model through _meta + try: + field = None + # Try to find the field in the model's _meta + for f in current_model._meta.get_fields(): + if hasattr(f, 'related_name') and f.related_name == relationship_name: + field = f + break + elif hasattr(f, 'name') and f.name == relationship_name: + field = f + break + + if field is None: + continue + + # Get the related model + if hasattr(field, 'related_model'): + related_model = field.related_model + elif hasattr(field, 'model'): + related_model = field.model + else: + continue + + related_model_name = related_model.__name__ + + # Find the reverse relationship name + reverse_relationship_name = self._find_reverse_relationship( + relationship_name, current_model, related_model + ) + + if reverse_relationship_name: + current_reverse_path.append(reverse_relationship_name) + # Add the reverse path + if related_model_name not in graph: + graph[related_model_name] = [] + + # The path back is just the reverse relationship name + graph[related_model_name] = current_reverse_path[::-1] # make a reverse copy + + current_model = related_model + + except Exception as e: + logger.info(f'error 2 > {e}') + # Skip problematic relationships + continue + + except Exception as e: + logger.info(f'error > {e}') + continue + + return graph + + def _find_reverse_relationship(self, original_relationship_name, in_model, for_model): + """ + Find the reverse relationship name from from_model to to_model. + + Args: + from_model: The model to search relationships from + to_model: The target model to find relationship to + original_relationship_name: The original relationship name for context + + Returns: + str or None: The reverse relationship name if found + """ + + print(f'reverse of {original_relationship_name} in = {in_model} for {for_model} ') + try: + for field in for_model._meta.get_fields(): + # Check ForeignKey, OneToOneField fields + # print(f'{for_model} : field name = {field.name} / field.related_model = {field.related_model == in_model}') + if hasattr(field, 'related_model') and field.related_model == in_model: + return field.name + ### possible improvements to do here if multiple relationships of the same type + + + # Check if this field has a related_name that matches our original relationship + # if hasattr(field, 'related_name') and field.related_name == original_relationship_name: + # # This is the reverse of our original relationship + # return field.name + # elif not hasattr(field, 'related_name') or field.related_name is None: + # default_name = f"{in_model._meta.related_query_name}" + # print(f'no related name: {default_name} / {original_relationship_name.rstrip('s')} ') + # if default_name == original_relationship_name.rstrip('s'): # Simple heuristic + # return field.name + + # Check reverse relationships + if hasattr(field, 'field') and hasattr(field.field, 'model'): + if field.field.model == in_model: + if field.get_accessor_name() == original_relationship_name: + return field.field.name + + except Exception as e: + print(f'!! ERROR = {e}') + pass + + return None + + def get_relationships(self, model_name: str) -> List[str]: + """ + Get the list of direct relationships for a given model name. + + Args: + model_name (str): The name of the model to look up + + Returns: + List[str]: List of relationship names for the model. + Returns empty list if model is not found. + """ + + return self._model_relationships.get(model_name, []) + + def get_relationship_paths(self, model_name: str) -> List[List[str]]: + """ + Get all relationship paths for a given model name. + This includes both direct relationships and reverse paths. + + Args: + model_name (str): The name of the model to look up + + Returns: + List[List[str]]: List of relationship paths (each path is a list of relationship names). + Returns empty list if model is not found. + """ + if not self._relationship_graph: + self._relationship_graph = self._build_relationship_graph() + logger.info(f'self._relationship_graph = {self._relationship_graph}') + + return self._relationship_graph.get(model_name, []) + + def get_relationship_graph(self) -> Dict[str, List[List[str]]]: + """ + Get the complete relationship graph. + + Returns: + Dict[str, List[List[str]]]: The complete relationship graph + """ + return self._relationship_graph.copy() + + +# Create a singleton instance to use throughout the application +sync_model_manager = SyncModelChildrenManager() diff --git a/sync/models/base.py b/sync/models/base.py index dc123a7..46955a1 100644 --- a/sync/models/base.py +++ b/sync/models/base.py @@ -1,11 +1,10 @@ from django.db import models from django.utils.timezone import now from django.conf import settings -from django.apps import apps from typing import List, Set +from ..model_manager import sync_model_manager -from collections import defaultdict import logging logger = logging.getLogger(__name__) @@ -41,8 +40,8 @@ class BaseModel(models.Model): def update_data_access_list(self): related_instances = self.sharing_related_instances() - data_access_ids = {instance.data_access_ids for instance in related_instances} - data_access_ids.update(self.data_access_ids) + data_access_ids = [instance.data_access_ids for instance in related_instances if isinstance(instance, BaseModel)] + data_access_ids.extend(self.data_access_ids) self.data_access_ids = data_access_ids # DataAccess = apps.get_model('sync', 'DataAccess') @@ -206,17 +205,21 @@ class BaseModel(models.Model): return instances def get_shared_children(self, processed_objects): + relationships_arrays = sync_model_manager.get_relationship_paths(self.__class__.__name__) + instances = [] - sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) - relationships = sync_models[self.__class__.__name__] - if relationships: - return self.get_shared_children_from_relationships(relationships, processed_objects) + if relationships_arrays: + for relationships in relationships_arrays: + children = self.get_shared_children_from_relationships(relationships, processed_objects) + instances.extend(children) else: - return self.get_recursive_children(processed_objects) + instances.extend(self.get_recursive_children(processed_objects)) + + return instances def get_shared_children_from_relationships(self, relationships, processed_objects): - # print(f'relationships = {relationships}') + print(f'>>> {self.__class__.__name__} : relationships = {relationships}') current = [self] for relationship in relationships: # print(f'> relationship = {relationship}') diff --git a/sync/models/data_access.py b/sync/models/data_access.py index 1e41874..b4019fa 100644 --- a/sync/models/data_access.py +++ b/sync/models/data_access.py @@ -69,8 +69,7 @@ class DataAccess(BaseModel): if model_class: try: obj = model_class.objects.get(id=self.model_id) - - related_instance = obj.sharing_related_instances() + related_instance = obj.sharing_related_instances() # here we want instances granted by the DataAccess, including SYNC_MODEL_CHILDREN_SHARING related_instance.append(obj) with transaction.atomic(): diff --git a/sync/registry.py b/sync/registry.py index 1f42797..881f2d6 100644 --- a/sync/registry.py +++ b/sync/registry.py @@ -2,29 +2,24 @@ from django.conf import settings from django.apps import apps from django.contrib.auth import get_user_model -from .models import BaseModel - import threading import logging - -from typing import List, Optional, Dict - +import importlib logger = logging.getLogger(__name__) -User = get_user_model() - class ModelRegistry: def __init__(self): self._registry = {} def load_sync_apps(self): + base_model = get_abstract_model_class('BaseModel') sync_apps = getattr(settings, 'SYNC_APPS', {}) for app_label, config in sync_apps.items(): app_models = apps.get_app_config(app_label).get_models() for model in app_models: if hasattr(model, '_meta') and not model._meta.abstract: - if issubclass(model, BaseModel) or model == User: + if issubclass(model, base_model) or model == get_user_model(): model_name = model.__name__ if self.should_sync_model(model_name, config): self.register(model) @@ -47,6 +42,10 @@ class ModelRegistry: # Global instance model_registry = ModelRegistry() +def get_abstract_model_class(model_name): + module = importlib.import_module('sync.models') + return getattr(module, model_name) + class DeviceRegistry: """Thread-safe registry to track device IDs associated with model instances.""" @@ -113,173 +112,3 @@ class RelatedUsersRegistry: # Global instance related_users_registry = RelatedUsersRegistry() - - -class SyncModelChildrenManager: - """ - Manager class for handling model children sharing configuration. - Reads the SYNC_MODEL_CHILDREN_SHARING setting once and builds a bidirectional - relationship graph for efficient lookup. - """ - - def __init__(self): - """Initialize the manager by reading the Django setting and building the relationship graph.""" - self._model_relationships = getattr( - settings, - 'SYNC_MODEL_CHILDREN_SHARING', - {} - ) - self._relationship_graph = self._build_relationship_graph() - logger.info(f'self._relationship_graph = {self._relationship_graph}') - - def _build_relationship_graph(self) -> Dict[str, List[List[str]]]: - """ - Build a bidirectional relationship graph. - - Returns: - Dict[str, List[List[str]]]: Dictionary where keys are model names and values - are lists of relationship paths (arrays of relationship names). - """ - graph = {} - - # Add direct relationships (original models to their children) - for model_name, relationships in self._model_relationships.items(): - if model_name not in graph: - graph[model_name] = [] - - # Add direct relationships as single-item arrays - for relationship in relationships: - graph[model_name].append([relationship]) - - # Build reverse relationships (children back to original models) - for original_model_name, relationships in self._model_relationships.items(): - try: - original_model = model_registry.get_model(original_model_name) - if original_model is None: - continue - - for relationship_name in relationships: - # Get the related model through _meta - try: - field = None - # Try to find the field in the model's _meta - for f in original_model._meta.get_fields(): - if hasattr(f, 'related_name') and f.related_name == relationship_name: - field = f - break - elif hasattr(f, 'name') and f.name == relationship_name: - field = f - break - - if field is None: - continue - - # Get the related model - if hasattr(field, 'related_model'): - related_model = field.related_model - elif hasattr(field, 'model'): - related_model = field.model - else: - continue - - related_model_name = related_model.__name__ - - # Find the reverse relationship name - reverse_relationship_name = self._find_reverse_relationship( - related_model, original_model, relationship_name - ) - - if reverse_relationship_name: - # Add the reverse path - if related_model_name not in graph: - graph[related_model_name] = [] - - # The path back is just the reverse relationship name - graph[related_model_name].append([reverse_relationship_name]) - - except Exception as e: - # Skip problematic relationships - continue - - except Exception as e: - # Skip problematic models - continue - - return graph - - def _find_reverse_relationship(self, from_model, to_model, original_relationship_name): - """ - Find the reverse relationship name from from_model to to_model. - - Args: - from_model: The model to search relationships from - to_model: The target model to find relationship to - original_relationship_name: The original relationship name for context - - Returns: - str or None: The reverse relationship name if found - """ - try: - for field in from_model._meta.get_fields(): - # Check ForeignKey, OneToOneField fields - if hasattr(field, 'related_model') and field.related_model == to_model: - # Check if this field has a related_name that matches our original relationship - if hasattr(field, 'related_name') and field.related_name == original_relationship_name: - # This is the reverse of our original relationship - return field.name - elif not hasattr(field, 'related_name') or field.related_name is None: - # Default reverse relationship name - default_name = f"{to_model._meta.model_name}" - if default_name == original_relationship_name.rstrip('s'): # Simple heuristic - return field.name - - # Check reverse relationships - if hasattr(field, 'field') and hasattr(field.field, 'model'): - if field.field.model == to_model: - if field.get_accessor_name() == original_relationship_name: - return field.field.name - - except Exception: - pass - - return None - - def get_relationships(self, model_name: str) -> List[str]: - """ - Get the list of direct relationships for a given model name. - - Args: - model_name (str): The name of the model to look up - - Returns: - List[str]: List of relationship names for the model. - Returns empty list if model is not found. - """ - return self._model_relationships.get(model_name, []) - - def get_relationship_paths(self, model_name: str) -> List[List[str]]: - """ - Get all relationship paths for a given model name. - This includes both direct relationships and reverse paths. - - Args: - model_name (str): The name of the model to look up - - Returns: - List[List[str]]: List of relationship paths (each path is a list of relationship names). - Returns empty list if model is not found. - """ - return self._relationship_graph.get(model_name, []) - - def get_relationship_graph(self) -> Dict[str, List[List[str]]]: - """ - Get the complete relationship graph. - - Returns: - Dict[str, List[List[str]]]: The complete relationship graph - """ - return self._relationship_graph.copy() - - -# Create a singleton instance to use throughout the application -sync_model_manager = SyncModelChildrenManager() diff --git a/tournaments/models/event.py b/tournaments/models/event.py index 6e446f4..229fa73 100644 --- a/tournaments/models/event.py +++ b/tournaments/models/event.py @@ -26,7 +26,7 @@ class Event(BaseModel): def save(self, *args, **kwargs): if self.creator: self.creator_full_name = self.creator.full_name() - super(Event, self).save(*args, **kwargs) + super().save(*args, **kwargs) def display_name(self): if self.club and self.club.name and self.name: From b503f7cb339a3b106a3d4c06933580002dbe50de Mon Sep 17 00:00:00 2001 From: Laurent Date: Sun, 22 Jun 2025 10:51:22 +0200 Subject: [PATCH 09/16] fix --- sync/model_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/model_manager.py b/sync/model_manager.py index 6fdff40..1c43964 100644 --- a/sync/model_manager.py +++ b/sync/model_manager.py @@ -88,7 +88,7 @@ class SyncModelChildrenManager: graph[related_model_name] = [] # The path back is just the reverse relationship name - graph[related_model_name] = current_reverse_path[::-1] # make a reverse copy + graph[related_model_name].append(current_reverse_path[::-1]) # make a reverse copy current_model = related_model From dd62e2f11ecbd8d6883de1dee8655826a1a5eb80 Mon Sep 17 00:00:00 2001 From: Laurent Date: Sun, 22 Jun 2025 14:50:51 +0200 Subject: [PATCH 10/16] fixes --- sync/models/base.py | 12 +++++++++--- sync/signals.py | 21 +-------------------- sync/views.py | 17 ++++++++++------- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/sync/models/base.py b/sync/models/base.py index 46955a1..cefd9f6 100644 --- a/sync/models/base.py +++ b/sync/models/base.py @@ -40,9 +40,15 @@ class BaseModel(models.Model): def update_data_access_list(self): related_instances = self.sharing_related_instances() - data_access_ids = [instance.data_access_ids for instance in related_instances if isinstance(instance, BaseModel)] - data_access_ids.extend(self.data_access_ids) - self.data_access_ids = data_access_ids + data_access_ids = {} + for instance in related_instances: + if isinstance(instance, BaseModel) and instance.data_access_ids: + data_access_ids.append(instance.data_access_ids) + + # print(f'related_instances = {related_instances}') + # data_access_ids = [instance.data_access_ids for instance in related_instances if isinstance(instance, BaseModel)] + # data_access_ids.extend(self.data_access_ids) + self.data_access_ids = list(data_access_ids) # DataAccess = apps.get_model('sync', 'DataAccess') # data_accesses = DataAccess.objects.filter(model_id__in=related_ids) diff --git a/sync/signals.py b/sync/signals.py index 7e275a8..0b9c6c4 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -11,7 +11,6 @@ from .ws_sender import websocket_sender from .registry import device_registry, related_users_registry import logging -import sys import traceback logger = logging.getLogger(__name__) @@ -34,29 +33,13 @@ def presave_handler(sender, instance, **kwargs): return users = related_users(instance) - related_users_registry.register(instance.id, users) if signal == pre_save: detect_foreign_key_changes_for_shared_instances(sender, instance) - sig_type = 'pre_save' - elif signal == pre_delete: - # if hasattr(instance, 'id'): - # try: - # data_access_list = DataAccess.objects.filter(model_id=instance.id) - # if data_access_list: - # logger.info(f'>>> {instance.__class__.__name__} {instance.id} : delete {data_access_list.count()} DataAccess') - # data_access_list.delete() - # except Exception as e: - # logger.info(f'*** ERRRRRRR: {e}') - # logger.info(traceback.format_exc()) - # raise - - sig_type = 'pre_delete' - # logger.info(f'* {sig_type} : {instance.__class__.__name__} > impacted users = {users}') except Exception as e: - logger.info(f'*** ERROR: {e}') + logger.info(f'*** presave_handler ERROR: {e}') raise @receiver([post_save, post_delete]) @@ -344,12 +327,10 @@ def related_users(instance): elif isinstance(instance, BaseModel): users.add(instance.related_user) data_access_list = DataAccess.objects.filter(id__in=instance.data_access_ids) - # print(f'instance = {instance.__class__.__name__}, data access count = {len(data_access_list)}') for data_access in data_access_list: users.add(data_access.related_user) users.update(data_access.shared_with.all()) - if isinstance(instance, DataAccess): users.update(instance.shared_with.all()) diff --git a/sync/views.py b/sync/views.py index c4dd8e3..100ff72 100644 --- a/sync/views.py +++ b/sync/views.py @@ -340,14 +340,17 @@ class LogProcessingResult: shared = defaultdict(dict) grants = defaultdict(dict) + try: # Process each grant instance - for model_name, instances in self.shared_instances.items(): - for model_id, instance in instances.items(): - serializer = get_serializer(instance, model_name) - shared[model_name][model_id] = serializer.data + for model_name, instances in self.shared_instances.items(): + for model_id, instance in instances.items(): + serializer = get_serializer(instance, model_name) + shared[model_name][model_id] = serializer.data - add_children_hierarchy(instance, grants) - add_parents_recursively(instance, grants) + add_children_hierarchy(instance, grants) + add_parents_recursively(instance, grants) + except Exception as e: + print(f'ERR = {e}') # Convert to lists for model_name in shared: @@ -363,7 +366,7 @@ class LogProcessingResult: revocations_parents_organizer = HierarchyOrganizer() # logger.info(f'$$$ process_revocations: {len(self.revoke_info)}') - sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) + # sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) # First, collect all revocations for model_name, items in self.revoke_info.items(): From 136a0697c41cc1fc9196410c3d117e37641d2cc4 Mon Sep 17 00:00:00 2001 From: Laurent Date: Sun, 22 Jun 2025 15:03:00 +0200 Subject: [PATCH 11/16] test DataAccess keeping --- sync/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/signals.py b/sync/signals.py index 0b9c6c4..ed28521 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -257,8 +257,8 @@ def process_foreign_key_changes(sender, instance, **kwargs): def delete_data_access_if_necessary(sender, instance, **kwargs): if not isinstance(instance, BaseModel): return - if hasattr(instance, 'id'): - DataAccess.objects.filter(model_id=instance.id).delete() + # if hasattr(instance, 'id'): + # DataAccess.objects.filter(model_id=instance.id).delete() @receiver(m2m_changed, sender=DataAccess.shared_with.through) def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs): From c8dd481ebd8cc86f741db227385986c59d238701 Mon Sep 17 00:00:00 2001 From: Laurent Date: Sun, 22 Jun 2025 15:06:10 +0200 Subject: [PATCH 12/16] test --- sync/signals.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sync/signals.py b/sync/signals.py index ed28521..a0bbcb0 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -257,8 +257,9 @@ def process_foreign_key_changes(sender, instance, **kwargs): def delete_data_access_if_necessary(sender, instance, **kwargs): if not isinstance(instance, BaseModel): return - # if hasattr(instance, 'id'): - # DataAccess.objects.filter(model_id=instance.id).delete() + if hasattr(instance, 'id'): + for data_access in DataAccess.objects.filter(model_id=instance.id): + data_access.create_revoke_access_log() @receiver(m2m_changed, sender=DataAccess.shared_with.through) def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs): From 9c121cb10642b9736d7879f2050f8f9902b0de62 Mon Sep 17 00:00:00 2001 From: Laurent Date: Tue, 24 Jun 2025 16:35:48 +0200 Subject: [PATCH 13/16] fix sharing issue --- sync/utils.py | 46 ++++++++++++++-- sync/views.py | 143 +++++++++++++++++++------------------------------- 2 files changed, 96 insertions(+), 93 deletions(-) diff --git a/sync/utils.py b/sync/utils.py index 07168bf..76afd26 100644 --- a/sync/utils.py +++ b/sync/utils.py @@ -1,7 +1,7 @@ import importlib -from django.apps import apps -from .registry import model_registry from collections import defaultdict +from .registry import model_registry +from .models import BaseModel, SideStoreModel import random import string @@ -50,7 +50,7 @@ class HierarchyOrganizer: def __init__(self): self.levels = [] # List of dictionaries, each representing a level self.item_levels = {} # Keep track of items and their levels: (model_name, id) -> level - self.sharing_related_instances = {} # Keep track of items and their levels: (model_name, id) -> level + self.children = set() def add_item(self, model_name, item_data, level): """ @@ -91,3 +91,43 @@ class HierarchyOrganizer: if cleaned_dict: cleaned_levels.append(cleaned_dict) return cleaned_levels + + def add_relations(self, instance): + self.add_related_parents(instance) + self.add_related_children(instance) + + def add_related_children(self, instance): + instance.get_shared_children(self.children) + + def grouped_children(self): + grouped = defaultdict(list) + for instance in self.children: + class_name = instance.__class__.__name__ + # serializer = get_serializer(instance, class_name) + grouped[class_name].append(instance.data_identifier_dict()) + return dict(grouped) + + def add_related_parents(self, instance, current_level=0): + """ + Recursively add all parents of an instance to the hierarchy organizer. + Parents are added at a higher level than their children. + """ + parent_models = instance.get_parents_by_model() + + for parent_model_name, parent in parent_models.items(): + if isinstance(parent, BaseModel): + store_id = None + if isinstance(parent, SideStoreModel): + store_id = parent.store_id + + parent_data = { + 'model_id': parent.id, + 'store_id': store_id + } + + # Add parent at the next level + # print(f'*** add parent: {parent_model_name}: {parent.id}') + self.add_item(parent_model_name, parent_data, current_level + 1) + + # Recursively process parent's parents + self.add_related_parents(parent, current_level + 1) diff --git a/sync/views.py b/sync/views.py index 100ff72..afb7ff6 100644 --- a/sync/views.py +++ b/sync/views.py @@ -30,6 +30,25 @@ logger = logging.getLogger(__name__) # class HierarchyApiView(APIView): +def instances_to_dict(instances, models_dict): + for instance in instances: + child_model_name = instance.__class__.__name__ + serializer = get_serializer(instance, child_model_name) + if child_model_name not in models_dict: + models_dict[child_model_name] = {} + + models_dict[child_model_name][instance.id] = serializer.data + return models_dict + +def instances_to_data_identifier_dict(instances, models_dict): + for instance in instances: + if isinstance(instance, BaseModel): + child_model_name = instance.__class__.__name__ + if child_model_name not in models_dict: + models_dict[child_model_name] = {} + models_dict[child_model_name][instance.id] = instance.data_identifier_dict() + return models_dict + def add_children_hierarchy(instance, models_dict): sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) if instance.__class__.__name__ in sync_models: @@ -79,30 +98,30 @@ def add_children_recursively(instance, models_dict): models_dict[child_model_name][child.id] = serializer.data add_children_recursively(child, models_dict) -def add_parents_with_hierarchy_organizer(instance, hierarchy_organizer, current_level=0): - """ - Recursively add all parents of an instance to the hierarchy organizer. - Parents are added at a higher level than their children. - """ - parent_models = instance.get_parents_by_model() +# def add_parents_with_hierarchy_organizer(instance, hierarchy_organizer, current_level=0): +# """ +# Recursively add all parents of an instance to the hierarchy organizer. +# Parents are added at a higher level than their children. +# """ +# parent_models = instance.get_parents_by_model() - for parent_model_name, parent in parent_models.items(): - if isinstance(parent, BaseModel): - store_id = None - if isinstance(parent, SideStoreModel): - store_id = parent.store_id +# for parent_model_name, parent in parent_models.items(): +# if isinstance(parent, BaseModel): +# store_id = None +# if isinstance(parent, SideStoreModel): +# store_id = parent.store_id - parent_data = { - 'model_id': parent.id, - 'store_id': store_id - } +# parent_data = { +# 'model_id': parent.id, +# 'store_id': store_id +# } - # Add parent at the next level - print(f'*** add parent: {parent_model_name}: {parent.id}') - hierarchy_organizer.add_item(parent_model_name, parent_data, current_level + 1) +# # Add parent at the next level +# print(f'*** add parent: {parent_model_name}: {parent.id}') +# hierarchy_organizer.add_item(parent_model_name, parent_data, current_level + 1) - # Recursively process parent's parents - add_parents_with_hierarchy_organizer(parent, hierarchy_organizer, current_level + 1) +# # Recursively process parent's parents +# add_parents_with_hierarchy_organizer(parent, hierarchy_organizer, current_level + 1) def add_parents_recursively(instance, dictionary): """ @@ -363,15 +382,11 @@ class LogProcessingResult: def process_revocations(self): """Process revocations and their hierarchies.""" revocations = defaultdict(list) - revocations_parents_organizer = HierarchyOrganizer() - - # logger.info(f'$$$ process_revocations: {len(self.revoke_info)}') - # sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) + revocated_relations_organizer = HierarchyOrganizer() # First, collect all revocations for model_name, items in self.revoke_info.items(): revocations[model_name].extend(items) - logger.info(f'$$$ process_revocations for {model_name}, items = {len(items)}') # Process parent hierarchies for each revoked item @@ -383,23 +398,20 @@ class LogProcessingResult: try: instance = model.objects.get(id=item['model_id']) logger.info(f'$$$ process revoked item parents of {model_name} : {item['model_id']}') - add_parents_with_hierarchy_organizer(instance, revocations_parents_organizer) - - # if instance.__class__.__name__ in sync_models: - # sharing_related_instances = sharing_related_instances(instance, True) - # logger.info(f'$$$ get shared instances: {len(sharing_related_instances)}') - # revocations = merge_dicts_dedup(revocations, sharing_related_instances) - # # revocations_parents_organizer.sharing_related_instances = instance.sharing_related_instances() + revocated_relations_organizer.add_relations(instance) except model.DoesNotExist: pass - return revocations, revocations_parents_organizer + children = revocated_relations_organizer.grouped_children() + merged_revocations = merge_dicts(revocations, children) + + return merged_revocations, revocated_relations_organizer def get_response_data(self): """Construct the complete response data structure.""" shared, grants = self.process_shared() - revocations, revocations_parents_organizer = self.process_revocations() + revocations, revocated_relations_organizer = self.process_revocations() # print(f'self.deletions = {dict(self.deletions)}') # print(f'self.shared_relationship_sets = {self.shared_relationship_sets}') @@ -413,7 +425,7 @@ class LogProcessingResult: "shared": dict(shared), "grants": dict(grants), "revocations": dict(revocations), - "revocation_parents": revocations_parents_organizer.get_organized_data(), + "revocated_relations": revocated_relations_organizer.get_organized_data(), "shared_relationship_sets": self.shared_relationship_sets, "shared_relationship_removals": self.shared_relationship_removals, "date": self.last_log_date @@ -472,60 +484,11 @@ class DataAccessViewSet(viewsets.ModelViewSet): return self.queryset.filter(Q(related_user=self.request.user) | Q(shared_with__in=[self.request.user])) return [] -def merge_dicts_dedup(dict1, dict2): - """Merge two dictionaries, combining arrays and removing duplicates""" - all_keys = set(dict1.keys()) | set(dict2.keys()) - - merged = {} - for key in all_keys: - arr1 = dict1.get(key, []) - arr2 = dict2.get(key, []) - # Convert to sets, union them, then back to list to remove duplicates - merged[key] = list(set(arr1) | set(arr2)) - - return merged - -def sharing_related_instances(instance, identifiers_only): - - sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) - # if self.__class__.__name__ in sync_models: - relationships = sync_models[instance.__class__.__name__] - # 'Match': {'team_scores', 'team_registration', 'player_registrations'} - - models_dict = defaultdict(dict) - - print(f'relationships = {relationships}') - current = [instance] - for relationship in relationships: - print(f'> relationship = {relationship}') - values = [] - for item in current: - value = getattr(item, relationship) - - if hasattr(value, 'all') and callable(value.all): - # This is a queryset from a reverse relationship - for related_obj in value.all(): - child_model_name = related_obj.__class__.__name__ - if identifiers_only: - models_dict[child_model_name].append(related_obj.data_identifier_dict()) - else: - serializer = get_serializer(related_obj, child_model_name) - models_dict[child_model_name].append(serializer.data) - # print(f'>>> 1 Added child for {relationship}: {child_model_name}') - values.extend(value.all()) - else: - # This is a single object - child_model_name = value.__class__.__name__ - if identifiers_only: - models_dict[child_model_name].append(value.data_identifier_dict()) - else: - serializer = get_serializer(value, child_model_name) - models_dict[child_model_name].append(serializer.data) +def merge_dicts(dict1, dict2): + result = defaultdict(list) - # serializer = get_serializer(value, child_model_name) - # models_dict[child_model_name][value.id] = serializer.data - # print(f'>>> 2 Added child for {relationship}: {child_model_name}') + for d in [dict1, dict2]: + for key, value in d.items(): + result[key].extend(value) - values.append(value) - current = values - return models_dict + return dict(result) From a32b2c2abc0e820e7db0b38377cd78a7534f791e Mon Sep 17 00:00:00 2001 From: Laurent Date: Thu, 26 Jun 2025 14:16:26 +0200 Subject: [PATCH 14/16] fix issue with null store_id --- sync/serializers.py | 6 ++++++ sync/utils.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sync/serializers.py b/sync/serializers.py index a48fa58..f37e6ba 100644 --- a/sync/serializers.py +++ b/sync/serializers.py @@ -6,3 +6,9 @@ class DataAccessSerializer(serializers.ModelSerializer): model = DataAccess fields = '__all__' read_only_fields = ['user'] + + def to_internal_value(self, data): + if 'store_id' in data and data['store_id'] is None: + data = data.copy() # Don't modify the original data + data['store_id'] = "" + return super().to_internal_value(data) diff --git a/sync/utils.py b/sync/utils.py index 76afd26..bcba5b4 100644 --- a/sync/utils.py +++ b/sync/utils.py @@ -103,7 +103,6 @@ class HierarchyOrganizer: grouped = defaultdict(list) for instance in self.children: class_name = instance.__class__.__name__ - # serializer = get_serializer(instance, class_name) grouped[class_name].append(instance.data_identifier_dict()) return dict(grouped) From cfbda0f0e669162b46a6c077976a3da0202e7b58 Mon Sep 17 00:00:00 2001 From: Laurent Date: Thu, 26 Jun 2025 14:20:42 +0200 Subject: [PATCH 15/16] remove junk and make store_id nullable for DataAccess --- .../0008_alter_dataaccess_store_id.py | 18 ++++++++++++++++++ sync/models/data_access.py | 2 +- sync/serializers.py | 6 ------ 3 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 sync/migrations/0008_alter_dataaccess_store_id.py diff --git a/sync/migrations/0008_alter_dataaccess_store_id.py b/sync/migrations/0008_alter_dataaccess_store_id.py new file mode 100644 index 0000000..d89a922 --- /dev/null +++ b/sync/migrations/0008_alter_dataaccess_store_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1 on 2025-06-26 12:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0007_dataaccess_store_id'), + ] + + operations = [ + migrations.AlterField( + model_name='dataaccess', + name='store_id', + field=models.CharField(blank=True, default='', max_length=100, null=True), + ), + ] diff --git a/sync/models/data_access.py b/sync/models/data_access.py index b4019fa..bc2507c 100644 --- a/sync/models/data_access.py +++ b/sync/models/data_access.py @@ -17,7 +17,7 @@ class DataAccess(BaseModel): shared_with = models.ManyToManyField(settings.AUTH_USER_MODEL, related_name='shared_data') model_name = models.CharField(max_length=50) model_id = models.UUIDField() - store_id = models.CharField(max_length=100, default="") # a value matching LeStorage directory sub-stores. Matches the name of the directory. + store_id = models.CharField(max_length=100, default="", blank=True, null=True) # a value matching LeStorage directory sub-stores. Matches the name of the directory. granted_at = models.DateTimeField(auto_now_add=True) def delete_dependencies(self): diff --git a/sync/serializers.py b/sync/serializers.py index f37e6ba..a48fa58 100644 --- a/sync/serializers.py +++ b/sync/serializers.py @@ -6,9 +6,3 @@ class DataAccessSerializer(serializers.ModelSerializer): model = DataAccess fields = '__all__' read_only_fields = ['user'] - - def to_internal_value(self, data): - if 'store_id' in data and data['store_id'] is None: - data = data.copy() # Don't modify the original data - data['store_id'] = "" - return super().to_internal_value(data) From e1e1dca3d6dd9755cc1b24b4afb141de5a37a03f Mon Sep 17 00:00:00 2001 From: Laurent Date: Thu, 26 Jun 2025 15:30:01 +0200 Subject: [PATCH 16/16] Fix issue --- sync/model_manager.py | 2 +- sync/models/base.py | 6 +++--- sync/signals.py | 2 +- sync/views.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sync/model_manager.py b/sync/model_manager.py index 1c43964..531ca14 100644 --- a/sync/model_manager.py +++ b/sync/model_manager.py @@ -176,7 +176,7 @@ class SyncModelChildrenManager: """ if not self._relationship_graph: self._relationship_graph = self._build_relationship_graph() - logger.info(f'self._relationship_graph = {self._relationship_graph}') + # logger.info(f'self._relationship_graph = {self._relationship_graph}') return self._relationship_graph.get(model_name, []) diff --git a/sync/models/base.py b/sync/models/base.py index cefd9f6..562dd56 100644 --- a/sync/models/base.py +++ b/sync/models/base.py @@ -40,10 +40,10 @@ class BaseModel(models.Model): def update_data_access_list(self): related_instances = self.sharing_related_instances() - data_access_ids = {} + data_access_ids = set() for instance in related_instances: if isinstance(instance, BaseModel) and instance.data_access_ids: - data_access_ids.append(instance.data_access_ids) + data_access_ids.update(instance.data_access_ids) # print(f'related_instances = {related_instances}') # data_access_ids = [instance.data_access_ids for instance in related_instances if isinstance(instance, BaseModel)] @@ -244,7 +244,7 @@ class BaseModel(models.Model): values.append(value) current = values - logger.info(f'+++ shared children = {processed_objects}') + # logger.info(f'+++ shared children = {processed_objects}') return processed_objects class SideStoreModel(BaseModel): diff --git a/sync/signals.py b/sync/signals.py index a0bbcb0..acce299 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -113,7 +113,7 @@ def save_model_log_if_possible(instance, signal, created): def save_model_log(users, model_operation, model_name, model_id, store_id): device_id = device_registry.get_device_id(model_id) - logger.info(f'*** creating ModelLogs for: {model_operation} {model_name} : {users}') + # logger.info(f'*** creating ModelLogs for: {model_operation} {model_name} : {users}') try: with transaction.atomic(): diff --git a/sync/views.py b/sync/views.py index afb7ff6..d8a27c9 100644 --- a/sync/views.py +++ b/sync/views.py @@ -55,7 +55,7 @@ def add_children_hierarchy(instance, models_dict): relationships = sync_models[instance.__class__.__name__] # 'Match': {'team_scores', 'team_registration', 'player_registrations'} - print(f'relationships = {relationships}') + # print(f'relationships = {relationships}') current = [instance] for relationship in relationships: print(f'> relationship = {relationship}') @@ -417,7 +417,7 @@ class LogProcessingResult: # print(f'self.shared_relationship_sets = {self.shared_relationship_sets}') # print(f'self.shared_relationship_removals = {self.shared_relationship_removals}') - logger.info('--------------------- SYNC') + # logger.info('--------------------- SYNC') return { "updates": dict(self.updates),