diff --git a/sync/migrations/0007_dataaccess_store_id.py b/sync/migrations/0007_dataaccess_store_id.py new file mode 100644 index 0000000..8113e21 --- /dev/null +++ b/sync/migrations/0007_dataaccess_store_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1 on 2025-06-12 13:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0006_alter_modellog_operation'), + ] + + operations = [ + migrations.AddField( + model_name='dataaccess', + name='store_id', + field=models.CharField(default='', max_length=100), + ), + ] diff --git a/sync/migrations/0008_alter_dataaccess_store_id.py b/sync/migrations/0008_alter_dataaccess_store_id.py new file mode 100644 index 0000000..d89a922 --- /dev/null +++ b/sync/migrations/0008_alter_dataaccess_store_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1 on 2025-06-26 12:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0007_dataaccess_store_id'), + ] + + operations = [ + migrations.AlterField( + model_name='dataaccess', + name='store_id', + field=models.CharField(blank=True, default='', max_length=100, null=True), + ), + ] diff --git a/sync/model_manager.py b/sync/model_manager.py new file mode 100644 index 0000000..531ca14 --- /dev/null +++ b/sync/model_manager.py @@ -0,0 +1,194 @@ +from django.conf import settings + +from typing import List, Dict +from .registry import model_registry + +import logging + +logger = logging.getLogger(__name__) + +class SyncModelChildrenManager: + """ + Manager class for handling model children sharing configuration. + Reads the SYNC_MODEL_CHILDREN_SHARING setting once and builds a bidirectional + relationship graph for efficient lookup. + """ + + def __init__(self): + """Initialize the manager by reading the Django setting and building the relationship graph.""" + self._model_relationships = getattr( + settings, + 'SYNC_MODEL_CHILDREN_SHARING', + {} + ) + self._relationship_graph = {} + + def _build_relationship_graph(self) -> Dict[str, List[List[str]]]: + """ + Build a bidirectional relationship graph. + + Returns: + Dict[str, List[List[str]]]: Dictionary where keys are model names and values + are lists of relationship paths (arrays of relationship names). + """ + graph = {} + + # Add direct relationships (original models to their children) + for model_name, relationships in self._model_relationships.items(): + if model_name not in graph: + graph[model_name] = [] + graph[model_name].append(relationships) + + # Build reverse relationships (children back to original models) + for original_model_name, relationships in self._model_relationships.items(): + + try: + current_model = model_registry.get_model(original_model_name) + if current_model is None: + print(f'missing {original_model_name}') + continue + + current_reverse_path = [] + + for relationship_name in relationships: + # Get the related model through _meta + try: + field = None + # Try to find the field in the model's _meta + for f in current_model._meta.get_fields(): + if hasattr(f, 'related_name') and f.related_name == relationship_name: + field = f + break + elif hasattr(f, 'name') and f.name == relationship_name: + field = f + break + + if field is None: + continue + + # Get the related model + if hasattr(field, 'related_model'): + related_model = field.related_model + elif hasattr(field, 'model'): + related_model = field.model + else: + continue + + related_model_name = related_model.__name__ + + # Find the reverse relationship name + reverse_relationship_name = self._find_reverse_relationship( + relationship_name, current_model, related_model + ) + + if reverse_relationship_name: + current_reverse_path.append(reverse_relationship_name) + # Add the reverse path + if related_model_name not in graph: + graph[related_model_name] = [] + + # The path back is just the reverse relationship name + graph[related_model_name].append(current_reverse_path[::-1]) # make a reverse copy + + current_model = related_model + + except Exception as e: + logger.info(f'error 2 > {e}') + # Skip problematic relationships + continue + + except Exception as e: + logger.info(f'error > {e}') + continue + + return graph + + def _find_reverse_relationship(self, original_relationship_name, in_model, for_model): + """ + Find the reverse relationship name from from_model to to_model. + + Args: + from_model: The model to search relationships from + to_model: The target model to find relationship to + original_relationship_name: The original relationship name for context + + Returns: + str or None: The reverse relationship name if found + """ + + print(f'reverse of {original_relationship_name} in = {in_model} for {for_model} ') + try: + for field in for_model._meta.get_fields(): + # Check ForeignKey, OneToOneField fields + # print(f'{for_model} : field name = {field.name} / field.related_model = {field.related_model == in_model}') + if hasattr(field, 'related_model') and field.related_model == in_model: + return field.name + ### possible improvements to do here if multiple relationships of the same type + + + # Check if this field has a related_name that matches our original relationship + # if hasattr(field, 'related_name') and field.related_name == original_relationship_name: + # # This is the reverse of our original relationship + # return field.name + # elif not hasattr(field, 'related_name') or field.related_name is None: + # default_name = f"{in_model._meta.related_query_name}" + # print(f'no related name: {default_name} / {original_relationship_name.rstrip('s')} ') + # if default_name == original_relationship_name.rstrip('s'): # Simple heuristic + # return field.name + + # Check reverse relationships + if hasattr(field, 'field') and hasattr(field.field, 'model'): + if field.field.model == in_model: + if field.get_accessor_name() == original_relationship_name: + return field.field.name + + except Exception as e: + print(f'!! ERROR = {e}') + pass + + return None + + def get_relationships(self, model_name: str) -> List[str]: + """ + Get the list of direct relationships for a given model name. + + Args: + model_name (str): The name of the model to look up + + Returns: + List[str]: List of relationship names for the model. + Returns empty list if model is not found. + """ + + return self._model_relationships.get(model_name, []) + + def get_relationship_paths(self, model_name: str) -> List[List[str]]: + """ + Get all relationship paths for a given model name. + This includes both direct relationships and reverse paths. + + Args: + model_name (str): The name of the model to look up + + Returns: + List[List[str]]: List of relationship paths (each path is a list of relationship names). + Returns empty list if model is not found. + """ + if not self._relationship_graph: + self._relationship_graph = self._build_relationship_graph() + # logger.info(f'self._relationship_graph = {self._relationship_graph}') + + return self._relationship_graph.get(model_name, []) + + def get_relationship_graph(self) -> Dict[str, List[List[str]]]: + """ + Get the complete relationship graph. + + Returns: + Dict[str, List[List[str]]]: The complete relationship graph + """ + return self._relationship_graph.copy() + + +# Create a singleton instance to use throughout the application +sync_model_manager = SyncModelChildrenManager() diff --git a/sync/models/base.py b/sync/models/base.py index 8668e9d..562dd56 100644 --- a/sync/models/base.py +++ b/sync/models/base.py @@ -1,9 +1,13 @@ from django.db import models from django.utils.timezone import now from django.conf import settings + from typing import List, Set +from ..model_manager import sync_model_manager + +import logging -from django.apps import apps +logger = logging.getLogger(__name__) class BaseModel(models.Model): creation_date = models.DateTimeField(default=now, editable=False) @@ -28,22 +32,28 @@ class BaseModel(models.Model): else: return None + def data_identifier_dict(self): + return { + 'model_id': self.id, + 'store_id': None + } + def update_data_access_list(self): - related_instances = self.related_instances() - related_ids = [ri.id for ri in related_instances] - related_ids.append(self.id) - - DataAccess = apps.get_model('sync', 'DataAccess') - data_accesses = DataAccess.objects.filter(model_id__in=related_ids) - for data_access in data_accesses: - self.add_data_access_relation(data_access) - - # add data_access to children who might not had the relationship - # if data_accesses: - # for child in self.get_children_by_model(): - # if len(child.data_access_ids) == 0: - # for data_access in data_accesses: - # self.add_data_access_relation(data_access) + related_instances = self.sharing_related_instances() + data_access_ids = set() + for instance in related_instances: + if isinstance(instance, BaseModel) and instance.data_access_ids: + data_access_ids.update(instance.data_access_ids) + + # print(f'related_instances = {related_instances}') + # data_access_ids = [instance.data_access_ids for instance in related_instances if isinstance(instance, BaseModel)] + # data_access_ids.extend(self.data_access_ids) + self.data_access_ids = list(data_access_ids) + + # DataAccess = apps.get_model('sync', 'DataAccess') + # data_accesses = DataAccess.objects.filter(model_id__in=related_ids) + # for data_access in data_accesses: + # self.add_data_access_relation(data_access) def add_data_access_relation(self, data_access): str_id = str(data_access.id) @@ -187,8 +197,64 @@ class BaseModel(models.Model): return None + def sharing_related_instances(self): + """ + Get all related instances (both children and parents) recursively + """ + instances = [] + processed_objects = set() + instances.extend(self.get_shared_children(processed_objects)) + + processed_objects = set() + instances.extend(self.get_recursive_parents(processed_objects)) + + return instances + + def get_shared_children(self, processed_objects): + relationships_arrays = sync_model_manager.get_relationship_paths(self.__class__.__name__) + instances = [] + + if relationships_arrays: + for relationships in relationships_arrays: + children = self.get_shared_children_from_relationships(relationships, processed_objects) + instances.extend(children) + else: + instances.extend(self.get_recursive_children(processed_objects)) + + return instances + + def get_shared_children_from_relationships(self, relationships, processed_objects): + + print(f'>>> {self.__class__.__name__} : relationships = {relationships}') + current = [self] + for relationship in relationships: + # print(f'> relationship = {relationship}') + values = [] + for item in current: + value = getattr(item, relationship) + + if hasattr(value, 'all') and callable(value.all): + # This is a queryset from a reverse relationship + for related_obj in value.all(): + processed_objects.add(related_obj) + values.extend(value.all()) + else: + processed_objects.add(value) + + values.append(value) + current = values + + # logger.info(f'+++ shared children = {processed_objects}') + return processed_objects + class SideStoreModel(BaseModel): store_id = models.CharField(max_length=100, default="") # a value matching LeStorage directory sub-stores. Matches the name of the directory. class Meta: abstract = True + + def data_identifier_dict(self): + return { + 'model_id': self.id, + 'store_id': self.store_id + } diff --git a/sync/models/data_access.py b/sync/models/data_access.py index 5a38ae3..bc2507c 100644 --- a/sync/models/data_access.py +++ b/sync/models/data_access.py @@ -1,13 +1,13 @@ from django.db import models from django.utils import timezone -# from django.apps import apps from django.core.exceptions import ObjectDoesNotExist from django.conf import settings +from django.db import transaction from ..registry import model_registry import uuid -from . import ModelLog, SideStoreModel, BaseModel +from . import ModelLog, BaseModel import logging logger = logging.getLogger(__name__) @@ -17,6 +17,7 @@ class DataAccess(BaseModel): shared_with = models.ManyToManyField(settings.AUTH_USER_MODEL, related_name='shared_data') model_name = models.CharField(max_length=50) model_id = models.UUIDField() + store_id = models.CharField(max_length=100, default="", blank=True, null=True) # a value matching LeStorage directory sub-stores. Matches the name of the directory. granted_at = models.DateTimeField(auto_now_add=True) def delete_dependencies(self): @@ -35,33 +36,31 @@ class DataAccess(BaseModel): """Create an access log for a list of users """ model_class = model_registry.get_model(self.model_name) if model_class: - try: - obj = model_class.objects.get(id=self.model_id) - store_id = None - if isinstance(obj, SideStoreModel): - store_id = obj.store_id - - for user in users: + for user in users: + logger.info(f'=== create ModelLog for: {operation} > {users}') + ModelLog.objects.create( + user=user, + model_id=self.model_id, + model_name=self.model_name, + operation=operation, + date=timezone.now(), + store_id=self.store_id + ) - logger.info(f'=== create ModelLog for: {operation} > {users}') - - existing_log = ModelLog.objects.filter(user=user, model_id=self.model_id, operation=operation).first() - if existing_log: - existing_log.date = timezone.now() - existing_log.model_operation = operation - existing_log.save() - else: - ModelLog.objects.create( - user=user, - model_id=self.model_id, - model_name=self.model_name, - operation=operation, - date=timezone.now(), - store_id=store_id - ) - except ObjectDoesNotExist: - logger.warn(f'!!! object does not exists any more: {self.model_name} : {self.model_id} : {operation}') - pass + # existing_log = ModelLog.objects.filter(user=user, model_id=self.model_id, operation=operation).first() + # if existing_log: + # existing_log.date = timezone.now() + # existing_log.model_operation = operation + # existing_log.save() + # else: + # ModelLog.objects.create( + # user=user, + # model_id=self.model_id, + # model_name=self.model_name, + # operation=operation, + # date=timezone.now(), + # store_id=self.store_id + # ) else: logger.warn(f'!!!model not found: {self.model_name}') @@ -70,12 +69,15 @@ class DataAccess(BaseModel): if model_class: try: obj = model_class.objects.get(id=self.model_id) - related_instance = obj.related_instances() + related_instance = obj.sharing_related_instances() # here we want instances granted by the DataAccess, including SYNC_MODEL_CHILDREN_SHARING related_instance.append(obj) - for instance in related_instance: - if isinstance(instance, BaseModel): - instance.add_data_access_relation(self) - instance.save() + + with transaction.atomic(): + for instance in related_instance: + logger.info(f'adds DataAccess to {instance.__class__.__name__}') + if isinstance(instance, BaseModel): + instance.add_data_access_relation(self) + instance.save() except ObjectDoesNotExist: pass @@ -86,9 +88,11 @@ class DataAccess(BaseModel): obj = model_class.objects.get(id=self.model_id) related_instance = obj.related_instances() related_instance.append(obj) - for instance in related_instance: - if isinstance(instance, BaseModel): - instance.remove_data_access_relation(self) - instance.save() + + with transaction.atomic(): + for instance in related_instance: + if isinstance(instance, BaseModel): + instance.remove_data_access_relation(self) + instance.save() except ObjectDoesNotExist: pass diff --git a/sync/registry.py b/sync/registry.py index f27b2e2..881f2d6 100644 --- a/sync/registry.py +++ b/sync/registry.py @@ -1,22 +1,25 @@ from django.conf import settings from django.apps import apps -from .models import BaseModel from django.contrib.auth import get_user_model + import threading +import logging +import importlib -User = get_user_model() +logger = logging.getLogger(__name__) class ModelRegistry: def __init__(self): self._registry = {} def load_sync_apps(self): + base_model = get_abstract_model_class('BaseModel') sync_apps = getattr(settings, 'SYNC_APPS', {}) for app_label, config in sync_apps.items(): app_models = apps.get_app_config(app_label).get_models() for model in app_models: if hasattr(model, '_meta') and not model._meta.abstract: - if issubclass(model, BaseModel) or model == User: + if issubclass(model, base_model) or model == get_user_model(): model_name = model.__name__ if self.should_sync_model(model_name, config): self.register(model) @@ -39,6 +42,10 @@ class ModelRegistry: # Global instance model_registry = ModelRegistry() +def get_abstract_model_class(model_name): + module = importlib.import_module('sync.models') + return getattr(module, model_name) + class DeviceRegistry: """Thread-safe registry to track device IDs associated with model instances.""" diff --git a/sync/signals.py b/sync/signals.py index e0c0fba..acce299 100644 --- a/sync/signals.py +++ b/sync/signals.py @@ -11,6 +11,7 @@ from .ws_sender import websocket_sender from .registry import device_registry, related_users_registry import logging +import traceback logger = logging.getLogger(__name__) @@ -21,28 +22,25 @@ User = get_user_model() @receiver([pre_save, pre_delete]) def presave_handler(sender, instance, **kwargs): - # some other classes are excluded in settings_app.py: SYNC_APPS - if not isinstance(instance, (BaseModel, User)): - return - - signal = kwargs.get('signal') - # avoid crash in manage.py createsuperuser + delete user in the admin - if isinstance(instance, User) and (instance._state.db is None or signal == pre_delete): - return - - users = related_users(instance) + try: + # some other classes are excluded in settings_app.py: SYNC_APPS + if not isinstance(instance, (BaseModel, User)) or isinstance(instance, DataAccess): + return - related_users_registry.register(instance.id, users) - # user_ids = [user.id for user in users] + signal = kwargs.get('signal') + # avoid crash in manage.py createsuperuser + delete user in the admin + if isinstance(instance, User) and (instance._state.db is None or signal == pre_delete): + return + users = related_users(instance) + related_users_registry.register(instance.id, users) - if signal == pre_save: - detect_foreign_key_changes_for_shared_instances(sender, instance) - sig_type = 'pre_save' - else: - sig_type = 'pre_delete' - logger.info(f'* {sig_type} : {instance.__class__.__name__} > impacted users = {users}') + if signal == pre_save: + detect_foreign_key_changes_for_shared_instances(sender, instance) + except Exception as e: + logger.info(f'*** presave_handler ERROR: {e}') + raise @receiver([post_save, post_delete]) def synchronization_notifications(sender, instance, created=False, **kwargs): @@ -56,47 +54,31 @@ def synchronization_notifications(sender, instance, created=False, **kwargs): if not isinstance(instance, BaseModel) and not isinstance(instance, User): return - process_foreign_key_changes(sender, instance, **kwargs) - - signal = kwargs.get('signal') - save_model_log_if_possible(instance, signal, created) - notify_impacted_users(instance) - - related_users_registry.unregister(instance.id) + try: + process_foreign_key_changes(sender, instance, **kwargs) + signal = kwargs.get('signal') + save_model_log_if_possible(instance, signal, created) + notify_impacted_users(instance) + related_users_registry.unregister(instance.id) + except Exception as e: + logger.info(f'*** ERROR2: {e}') + logger.info(traceback.format_exc()) + raise def notify_impacted_users(instance): - # print(f'*** notify_impacted_users for instance: {instance}') - # user_ids = set() - # # add impacted users - # if isinstance(instance, User): - # user_ids.add(instance.id) - # elif isinstance(instance, BaseModel): - # owner = instance.last_updated_by - # if owner: - # user_ids.add(owner.id) - - # if isinstance(instance, BaseModel): - # if hasattr(instance, '_users_to_notify'): - # user_ids.update(instance._users_to_notify) - # else: - # print('no users to notify') - device_id = device_registry.get_device_id(instance.id) users = related_users_registry.get_users(instance.id) if users: user_ids = [user.id for user in users] websocket_sender.send_message(user_ids, device_id) - # print(f'notify device: {device_id}, users = {user_ids}') - # for user_id in user_ids: - # websocket_sender.send_user_message(user_id, device_id) device_registry.unregister(instance.id) def save_model_log_if_possible(instance, signal, created): users = related_users_registry.get_users(instance.id) - logger.debug(f'*** save_model_log_if_possible >>> users from registry = {users}, instance = {instance}') + # logger.info(f'*** save_model_log_if_possible >>> users from registry = {users}, instance = {instance}') if not users: logger.warning(f'!!! Registry returned empty users for instance {instance.id} ({instance.__class__.__name__})') @@ -118,8 +100,8 @@ def save_model_log_if_possible(instance, signal, created): if isinstance(instance, SideStoreModel): store_id = instance.store_id - if operation == ModelOperation.DELETE: # delete now unnecessary logs - ModelLog.objects.filter(model_id=instance.id).delete() + # if operation == ModelOperation.DELETE: # delete now unnecessary logs + # ModelLog.objects.filter(model_id=instance.id).delete() # user_ids = [user.id for user in users] # # print(f'users to notify: {user_ids}') @@ -130,16 +112,14 @@ def save_model_log_if_possible(instance, signal, created): logger.info(f'!!! Model Log could not be created because no linked user could be found: {instance.__class__.__name__} {instance}, {signal}') def save_model_log(users, model_operation, model_name, model_id, store_id): - device_id = device_registry.get_device_id(model_id) - - logger.info(f'*** creating ModelLogs for: {model_operation} {model_name} : {users}') + # logger.info(f'*** creating ModelLogs for: {model_operation} {model_name} : {users}') try: with transaction.atomic(): created_logs = [] for user in users: - logger.debug(f'Creating ModelLog for user {user.id} ({user.username})') + # logger.info(f'Creating ModelLog for user {user.id} - user exists: {User.objects.filter(id=user.id).exists()}') model_log = ModelLog( user=user, operation=model_operation, @@ -149,20 +129,19 @@ def save_model_log(users, model_operation, model_name, model_id, store_id): device_id=device_id ) model_log.save() + # logger.info(f'ModelLog saved with ID: {model_log.id}') created_logs.append(model_log.id) - logger.debug(f'Successfully created ModelLog {model_log.id}') - logger.info(f'*** Successfully created {len(created_logs)} ModelLogs: {created_logs}') + # Immediate verification within transaction + immediate_count = ModelLog.objects.filter(id__in=created_logs).count() + # logger.info(f'*** Within transaction: Created {len(created_logs)}, found {immediate_count}') - # Verify ModelLogs were actually persisted - persisted_count = ModelLog.objects.filter(id__in=created_logs).count() - if persisted_count != len(created_logs): - logger.error(f'*** PERSISTENCE VERIFICATION FAILED! Created {len(created_logs)} ModelLogs but only {persisted_count} were persisted to database') - else: - logger.debug(f'*** PERSISTENCE VERIFIED: All {persisted_count} ModelLogs successfully persisted') + # Verification after transaction commits + persisted_count = ModelLog.objects.filter(id__in=created_logs).count() + # logger.info(f'*** After transaction: Created {len(created_logs)}, persisted {persisted_count}') except Exception as e: - logger.error(f'*** FAILED to create ModelLogs for: {model_operation} {model_name}, users: {[u.id for u in users]}, error: {e}', exc_info=True) + logger.error(f'*** Exception during ModelLog creation: {e}', exc_info=True) raise # with transaction.atomic(): @@ -206,7 +185,6 @@ def detect_foreign_key_changes_for_shared_instances(sender, instance): return data_access_list = related_data_access(instance) - # print(f'FK change > DA count = {len(data_access_list)}') if data_access_list: try: old_instance = sender.objects.get(pk=instance.pk) @@ -280,7 +258,8 @@ def delete_data_access_if_necessary(sender, instance, **kwargs): if not isinstance(instance, BaseModel): return if hasattr(instance, 'id'): - DataAccess.objects.filter(model_id=instance.id).delete() + for data_access in DataAccess.objects.filter(model_id=instance.id): + data_access.create_revoke_access_log() @receiver(m2m_changed, sender=DataAccess.shared_with.through) def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs): @@ -306,26 +285,41 @@ def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs): @receiver(post_save, sender=DataAccess) def data_access_post_save(sender, instance, **kwargs): - instance.add_references() # create DataAccess references on hierarchy + try: + instance.add_references() # create DataAccess references on hierarchy - if instance.related_user: - evaluate_if_user_should_sync(instance.related_user) + if instance.related_user: + evaluate_if_user_should_sync(instance.related_user) + except Exception as e: + logger.info(f'*** ERROR3: {e}') + logger.info(traceback.format_exc()) + raise @receiver(pre_delete, sender=DataAccess) def revoke_access_after_delete(sender, instance, **kwargs): - instance.cleanup_references() - instance.create_revoke_access_log() - related_users_registry.register(instance.id, instance.shared_with.all()) + try: + instance.cleanup_references() + instance.create_revoke_access_log() + related_users_registry.register(instance.id, instance.shared_with.all()) - instance._user = instance.related_user + instance._user = instance.related_user + except Exception as e: + logger.info(f'*** ERROR4: {e}') + logger.info(traceback.format_exc()) + raise @receiver(post_delete, sender=DataAccess) def data_access_post_delete(sender, instance, **kwargs): - notify_impacted_users(instance) + try: + notify_impacted_users(instance) - if not hasattr(instance, '_user') or not instance._user: - return - evaluate_if_user_should_sync(instance._user) + if not hasattr(instance, '_user') or not instance._user: + return + evaluate_if_user_should_sync(instance._user) + except Exception as e: + logger.info(f'*** ERROR5: {e}') + logger.info(traceback.format_exc()) + raise def related_users(instance): users = set() @@ -334,12 +328,10 @@ def related_users(instance): elif isinstance(instance, BaseModel): users.add(instance.related_user) data_access_list = DataAccess.objects.filter(id__in=instance.data_access_ids) - # print(f'instance = {instance.__class__.__name__}, data access count = {len(data_access_list)}') for data_access in data_access_list: users.add(data_access.related_user) users.update(data_access.shared_with.all()) - if isinstance(instance, DataAccess): users.update(instance.shared_with.all()) @@ -370,9 +362,14 @@ def evaluate_if_user_should_sync(user): @receiver(post_save, sender=Device) def device_created(sender, instance, **kwargs): - if not instance.user: - return - evaluate_if_user_should_sync(instance.user) + try: + if not instance.user: + return + evaluate_if_user_should_sync(instance.user) + except Exception as e: + logger.info(f'*** ERROR6: {e}') + logger.info(traceback.format_exc()) + raise @receiver(pre_delete, sender=Device) def device_pre_delete(sender, instance, **kwargs): @@ -380,6 +377,11 @@ def device_pre_delete(sender, instance, **kwargs): @receiver(post_delete, sender=Device) def device_post_delete(sender, instance, **kwargs): - if not hasattr(instance, '_user') or not instance._user: - return - evaluate_if_user_should_sync(instance._user) + try: + if not hasattr(instance, '_user') or not instance._user: + return + evaluate_if_user_should_sync(instance._user) + except Exception as e: + logger.info(f'*** ERROR7: {e}') + logger.info(traceback.format_exc()) + raise diff --git a/sync/utils.py b/sync/utils.py index ee7ff80..bcba5b4 100644 --- a/sync/utils.py +++ b/sync/utils.py @@ -1,7 +1,7 @@ import importlib -from django.apps import apps -from .registry import model_registry from collections import defaultdict +from .registry import model_registry +from .models import BaseModel, SideStoreModel import random import string @@ -50,6 +50,7 @@ class HierarchyOrganizer: def __init__(self): self.levels = [] # List of dictionaries, each representing a level self.item_levels = {} # Keep track of items and their levels: (model_name, id) -> level + self.children = set() def add_item(self, model_name, item_data, level): """ @@ -90,3 +91,42 @@ class HierarchyOrganizer: if cleaned_dict: cleaned_levels.append(cleaned_dict) return cleaned_levels + + def add_relations(self, instance): + self.add_related_parents(instance) + self.add_related_children(instance) + + def add_related_children(self, instance): + instance.get_shared_children(self.children) + + def grouped_children(self): + grouped = defaultdict(list) + for instance in self.children: + class_name = instance.__class__.__name__ + grouped[class_name].append(instance.data_identifier_dict()) + return dict(grouped) + + def add_related_parents(self, instance, current_level=0): + """ + Recursively add all parents of an instance to the hierarchy organizer. + Parents are added at a higher level than their children. + """ + parent_models = instance.get_parents_by_model() + + for parent_model_name, parent in parent_models.items(): + if isinstance(parent, BaseModel): + store_id = None + if isinstance(parent, SideStoreModel): + store_id = parent.store_id + + parent_data = { + 'model_id': parent.id, + 'store_id': store_id + } + + # Add parent at the next level + # print(f'*** add parent: {parent_model_name}: {parent.id}') + self.add_item(parent_model_name, parent_data, current_level + 1) + + # Recursively process parent's parents + self.add_related_parents(parent, current_level + 1) diff --git a/sync/views.py b/sync/views.py index 3f27041..d8a27c9 100644 --- a/sync/views.py +++ b/sync/views.py @@ -24,15 +24,38 @@ from .models import ModelLog, BaseModel, SideStoreModel, DataAccess from .registry import model_registry, device_registry from .ws_sender import websocket_sender +import logging + +logger = logging.getLogger(__name__) + # class HierarchyApiView(APIView): +def instances_to_dict(instances, models_dict): + for instance in instances: + child_model_name = instance.__class__.__name__ + serializer = get_serializer(instance, child_model_name) + if child_model_name not in models_dict: + models_dict[child_model_name] = {} + + models_dict[child_model_name][instance.id] = serializer.data + return models_dict + +def instances_to_data_identifier_dict(instances, models_dict): + for instance in instances: + if isinstance(instance, BaseModel): + child_model_name = instance.__class__.__name__ + if child_model_name not in models_dict: + models_dict[child_model_name] = {} + models_dict[child_model_name][instance.id] = instance.data_identifier_dict() + return models_dict + def add_children_hierarchy(instance, models_dict): sync_models = getattr(settings, 'SYNC_MODEL_CHILDREN_SHARING', {}) if instance.__class__.__name__ in sync_models: relationships = sync_models[instance.__class__.__name__] # 'Match': {'team_scores', 'team_registration', 'player_registrations'} - print(f'relationships = {relationships}') + # print(f'relationships = {relationships}') current = [instance] for relationship in relationships: print(f'> relationship = {relationship}') @@ -75,30 +98,30 @@ def add_children_recursively(instance, models_dict): models_dict[child_model_name][child.id] = serializer.data add_children_recursively(child, models_dict) -def add_parents_with_hierarchy_organizer(instance, hierarchy_organizer, current_level=0): - """ - Recursively add all parents of an instance to the hierarchy organizer. - Parents are added at a higher level than their children. - """ - parent_models = instance.get_parents_by_model() +# def add_parents_with_hierarchy_organizer(instance, hierarchy_organizer, current_level=0): +# """ +# Recursively add all parents of an instance to the hierarchy organizer. +# Parents are added at a higher level than their children. +# """ +# parent_models = instance.get_parents_by_model() - for parent_model_name, parent in parent_models.items(): - if isinstance(parent, BaseModel): - store_id = None - if isinstance(parent, SideStoreModel): - store_id = parent.store_id +# for parent_model_name, parent in parent_models.items(): +# if isinstance(parent, BaseModel): +# store_id = None +# if isinstance(parent, SideStoreModel): +# store_id = parent.store_id - parent_data = { - 'model_id': parent.id, - 'store_id': store_id - } +# parent_data = { +# 'model_id': parent.id, +# 'store_id': store_id +# } - # Add parent at the next level - print(f'*** add parent: {parent_model_name}: {parent.id}') - hierarchy_organizer.add_item(parent_model_name, parent_data, current_level + 1) +# # Add parent at the next level +# print(f'*** add parent: {parent_model_name}: {parent.id}') +# hierarchy_organizer.add_item(parent_model_name, parent_data, current_level + 1) - # Recursively process parent's parents - add_parents_with_hierarchy_organizer(parent, hierarchy_organizer, current_level + 1) +# # Recursively process parent's parents +# add_parents_with_hierarchy_organizer(parent, hierarchy_organizer, current_level + 1) def add_parents_recursively(instance, dictionary): """ @@ -336,14 +359,17 @@ class LogProcessingResult: shared = defaultdict(dict) grants = defaultdict(dict) + try: # Process each grant instance - for model_name, instances in self.shared_instances.items(): - for model_id, instance in instances.items(): - serializer = get_serializer(instance, model_name) - shared[model_name][model_id] = serializer.data + for model_name, instances in self.shared_instances.items(): + for model_id, instance in instances.items(): + serializer = get_serializer(instance, model_name) + shared[model_name][model_id] = serializer.data - add_children_hierarchy(instance, grants) - add_parents_recursively(instance, grants) + add_children_hierarchy(instance, grants) + add_parents_recursively(instance, grants) + except Exception as e: + print(f'ERR = {e}') # Convert to lists for model_name in shared: @@ -356,44 +382,50 @@ class LogProcessingResult: def process_revocations(self): """Process revocations and their hierarchies.""" revocations = defaultdict(list) - revocations_parents_organizer = HierarchyOrganizer() - - # print(f'*** process_revocations: {len(self.revoke_info)}') + revocated_relations_organizer = HierarchyOrganizer() # First, collect all revocations for model_name, items in self.revoke_info.items(): revocations[model_name].extend(items) - - # print(f'*** process_revocations for {model_name}') + logger.info(f'$$$ process_revocations for {model_name}, items = {len(items)}') # Process parent hierarchies for each revoked item model = model_registry.get_model(model_name) for item in items: + + logger.info(f'$$$ item revoked = {item}') + try: instance = model.objects.get(id=item['model_id']) - # print(f'*** process revoked item parents of {model_name} : {item['model_id']}') - add_parents_with_hierarchy_organizer(instance, revocations_parents_organizer) + logger.info(f'$$$ process revoked item parents of {model_name} : {item['model_id']}') + revocated_relations_organizer.add_relations(instance) + except model.DoesNotExist: pass - return revocations, revocations_parents_organizer + children = revocated_relations_organizer.grouped_children() + merged_revocations = merge_dicts(revocations, children) + + return merged_revocations, revocated_relations_organizer def get_response_data(self): """Construct the complete response data structure.""" shared, grants = self.process_shared() - revocations, revocations_parents_organizer = self.process_revocations() + revocations, revocated_relations_organizer = self.process_revocations() # print(f'self.deletions = {dict(self.deletions)}') # print(f'self.shared_relationship_sets = {self.shared_relationship_sets}') # print(f'self.shared_relationship_removals = {self.shared_relationship_removals}') + # logger.info('--------------------- SYNC') + return { "updates": dict(self.updates), "deletions": dict(self.deletions), "shared": dict(shared), "grants": dict(grants), "revocations": dict(revocations), - "revocation_parents": revocations_parents_organizer.get_organized_data(), + "revocated_relations": revocated_relations_organizer.get_organized_data(), "shared_relationship_sets": self.shared_relationship_sets, "shared_relationship_removals": self.shared_relationship_removals, "date": self.last_log_date @@ -451,3 +483,12 @@ class DataAccessViewSet(viewsets.ModelViewSet): if self.request.user: return self.queryset.filter(Q(related_user=self.request.user) | Q(shared_with__in=[self.request.user])) return [] + +def merge_dicts(dict1, dict2): + result = defaultdict(list) + + for d in [dict1, dict2]: + for key, value in d.items(): + result[key].extend(value) + + return dict(result) diff --git a/tournaments/models/event.py b/tournaments/models/event.py index 6e446f4..229fa73 100644 --- a/tournaments/models/event.py +++ b/tournaments/models/event.py @@ -26,7 +26,7 @@ class Event(BaseModel): def save(self, *args, **kwargs): if self.creator: self.creator_full_name = self.creator.full_name() - super(Event, self).save(*args, **kwargs) + super().save(*args, **kwargs) def display_name(self): if self.club and self.club.name and self.name: diff --git a/tournaments/models/match.py b/tournaments/models/match.py index ede4e07..33cf642 100644 --- a/tournaments/models/match.py +++ b/tournaments/models/match.py @@ -1,7 +1,9 @@ from django.db import models -# from tournaments.models import group_stage -from . import TournamentSubModel, Round, GroupStage, FederalMatchCategory from django.utils import timezone, formats +from django.core.exceptions import ObjectDoesNotExist + +from . import TournamentSubModel, Round, GroupStage, FederalMatchCategory + from datetime import datetime, timedelta import uuid @@ -90,14 +92,16 @@ class Match(TournamentSubModel): return '--' def stage_name(self): - if self.name: - return self.name - elif self.round: - return self.round.name() - elif self.group_stage: - return self.group_stage.display_name() - else: - return '--' + try: + if self.name: + return self.name + elif self.round: + return self.round.name() + elif self.group_stage: + return self.group_stage.display_name() + except ObjectDoesNotExist: + pass + return "--" def get_previous_round(self): # Calculate the next index diff --git a/tournaments/models/team_score.py b/tournaments/models/team_score.py index 14765a6..b4845ff 100644 --- a/tournaments/models/team_score.py +++ b/tournaments/models/team_score.py @@ -1,4 +1,6 @@ from django.db import models +from django.core.exceptions import ObjectDoesNotExist + from . import TournamentSubModel, Match, TeamRegistration, FederalMatchCategory import uuid from .match import Team # Import Team only when needed @@ -15,10 +17,14 @@ class TeamScore(TournamentSubModel): pass def __str__(self): - if self.match: - return f"{self.match.stage_name()} #{self.match.index}: {self.player_names()}" - else: - return "Empty" + try: + if self.match and self.team_registration: + return f"{self.match.stage_name()} #{self.match.index}: {self.player_names()}" + if self.match: + return f"{self.match.stage_name()} #{self.match.index}" + except ObjectDoesNotExist: + pass + return "--" def get_tournament(self): # mandatory method for TournamentSubModel if self.team_registration: @@ -48,14 +54,16 @@ class TeamScore(TournamentSubModel): # return None def player_names(self): - if self.team_registration: - if self.team_registration.name: - return self.team_registration.name - else: - names = self.team_registration.team_names() - return " - ".join(names) - else: - return "--" + try: + if self.team_registration: # this can cause an exception when deleted + if self.team_registration.name: + return self.team_registration.name + else: + names = self.team_registration.team_names() + return " - ".join(names) + except TeamRegistration.DoesNotExist: + pass + return "--" def shortened_team_names(self, forced=False): names = []