Sync fixes

api/urls.py

@@ -33,7 +33,7 @@ urlpatterns = [
     path('', include(router.urls)),
 
     path('sync-data/', SynchronizationApi.as_view(), name="data"),
-    path('data-access/', UserDataAccessApi.as_view(), name="user-data-access"),
+    path('data-access-content/', UserDataAccessApi.as_view(), name="data-access-content"),
 
     path('api-token-auth/', obtain_auth_token, name='api_token_auth'),
     path("user-by-token/", views.user_by_token, name="user_by_token"),

sync/signals.py

@@ -33,13 +33,13 @@ def device_post_delete(sender, instance, **kwargs):
 
 ### Sync
 
-@receiver(pre_save)
+@receiver([pre_save, pre_delete])
 def presave_handler(sender, instance, **kwargs):
     synchronization_prepare(sender, instance, **kwargs)
 
 def synchronization_prepare(sender, instance, **kwargs):
 
-    print(f'*** synchronization_prepare for instance: {instance}')
+    # print(f'*** synchronization_prepare for instance: {instance}')
     signal = kwargs.get('signal')
 
     # avoid crash in manage.py createsuperuser + delete user in the admin
@@ -51,7 +51,7 @@ def synchronization_prepare(sender, instance, **kwargs):
         return
 
     users = related_users(instance)
-    print(f'* impacted users = {users}')
+    # print(f'* impacted users = {users}')
     related_users_registry.register(instance.id, users)
     # user_ids = [user.id for user in users]
 
@@ -79,7 +79,7 @@ def synchronization_notifications(sender, instance, created=False, **kwargs):
     related_users_registry.unregister(instance.id)
 
 def notify_impacted_users(instance):
-    print(f'*** notify_impacted_users for instance: {instance}')
+    # print(f'*** notify_impacted_users for instance: {instance}')
     # user_ids = set()
     # # add impacted users
     # if isinstance(instance, User):
@@ -100,7 +100,7 @@ def notify_impacted_users(instance):
 
     if users:
         user_ids = [user.id for user in users]
-        print(f'notify device: {device_id}, users = {user_ids}')
+        # print(f'notify device: {device_id}, users = {user_ids}')
         for user_id in user_ids:
             websocket_sender.send_user_message(user_id, device_id)
 
@@ -109,7 +109,7 @@ def notify_impacted_users(instance):
 def save_model_log_if_possible(instance, signal, created):
 
     users = related_users_registry.get_users(instance.id)
-    print(f'*** save_model_log >>> users = {users}, instance = {instance}')
+    # print(f'*** save_model_log >>> users = {users}, instance = {instance}')
     if users:
         if signal == post_save or signal == pre_save:
             if created:
@@ -193,6 +193,9 @@ def detect_foreign_key_changes(sender, instance):
                 old_value = getattr(old_instance, field.name, None)
                 new_value = getattr(instance, field.name, None)
                 if old_value != new_value:
+                    if not hasattr(instance, '_fk_changes'):
+                        instance._fk_changes = []
+
                     instance._fk_changes.append({
                                            'data_access_list': data_access_list,
                                            'old_value': old_value,
@@ -227,7 +230,7 @@ def delete_data_access_if_necessary(sender, instance, **kwargs):
 @receiver(m2m_changed, sender=DataAccess.shared_with.through)
 def handle_shared_with_changes(sender, instance, action, pk_set, **kwargs):
 
-    print(f'm2m changed = {pk_set}')
+    # print(f'm2m changed = {pk_set}')
     users = User.objects.filter(id__in=pk_set)
 
     if action == "post_add":
@@ -269,13 +272,13 @@ def related_users(instance):
         users.add(instance)
     elif isinstance(instance, BaseModel):
         users.add(instance.related_user)
-        users.add(instance.last_updated_by)
+        # users.add(instance.last_updated_by)
 
         # look in hierarchy
         related_instances = instance.related_instances()
         # print(f'related_instances = {related_instances}')
-        related_users = [ri.related_user for ri in related_instances if isinstance(ri, BaseModel)]
-        users.update(related_users)
+        # related_users = [ri.related_user for ri in related_instances if isinstance(ri, BaseModel)]
+        # users.update(related_users)
 
         # look in related DataAccess
         data_access_list = instances_related_data_access(instance, related_instances)

sync/views.py

@@ -182,42 +182,149 @@ class SynchronizationApi(HierarchyApiView):
             'results': results
         }, status=207)  # Multi-Status
 
+    # def get(self, request, *args, **kwargs):
+    #     last_update_str = request.query_params.get('last_update')
+    #     device_id = request.query_params.get('device_id')
+
+    #     # print(f'last_update_str = {last_update_str}')
+    #     decoded_last_update = unquote(last_update_str)  # Decodes %2B into +
+
+    #     # print(f'last_update_str = {last_update_str}')
+    #     # print(f'decoded_last_update = {decoded_last_update}')
+    #     if not decoded_last_update:
+    #         return Response({"error": "last_update parameter is required"}, status=status.HTTP_400_BAD_REQUEST)
+    #     try:
+    #         last_update = timezone.datetime.fromisoformat(decoded_last_update)
+    #     except ValueError:
+    #         return Response({"error": f"Invalid date format for last_update: {decoded_last_update}"}, status=status.HTTP_400_BAD_REQUEST)
+
+    #     # print(f'/data GET: {last_update}')
+
+    #     logs = self.query_model_logs(last_update, request.user, device_id)
+    #     print(f'>>> log count = {len(logs)}')
+
+    #     updates = defaultdict(dict)
+    #     deletions = defaultdict(list)
+    #     grants = defaultdict(dict)
+    #     revocations = defaultdict(list)
+    #     revocations_parents_organizer = HierarchyOrganizer()
+
+    #     # revocated_parents = defaultdict(dict)
+
+    #     last_log_date = None
+    #     for log in logs:
+
+    #         # log.retrieved()
+    #         # log.save()
+
+    #         # print(f'log date = {log.date}')
+    #         last_log_date = log.date
+    #         try:
+    #             if log.operation in ['POST', 'PUT']:
+    #                 data = get_serialized_data(log.model_name, log.model_id)
+    #                 updates[log.model_name][log.model_id] = data
+    #             elif log.operation == 'DELETE':
+    #                 deletions[log.model_name].append({'model_id': log.model_id, 'store_id': log.store_id})
+    #             elif log.operation == 'GRANT_ACCESS':
+
+    #                 model = model_registry.get_model(log.model_name)
+    #                 instance = model.objects.get(id=log.model_id)
+    #                 serializer = get_serializer(instance, log.model_name)
+
+    #                 grants[log.model_name][log.model_id] = serializer.data
+    #                 self.add_children_recursively(instance, grants)
+    #                 self.add_parents_recursively(instance, grants)
+    #             elif log.operation == 'REVOKE_ACCESS':
+    #                 print(f'revoke access {log.model_id} - {log.store_id}')
+    #                 revocations[log.model_name].append({
+    #                     'model_id': log.model_id,
+    #                     'store_id': log.store_id
+    #                 })
+
+    #                 # Get the model instance and add its parents to hierarchy
+    #                 model = model_registry.get_model(log.model_name)
+    #                 try:
+    #                     instance = model.objects.get(id=log.model_id)
+    #                     self.add_parents_with_hierarchy_organizer(instance, revocations_parents_organizer)
+    #                 except model.DoesNotExist:
+    #                     pass
+    #         except ObjectDoesNotExist:
+    #             pass
+
+    #     # Convert updates dict to list for each model
+    #     for model_name in updates:
+    #         updates[model_name] = list(updates[model_name].values())
+
+    #     # Convert deletions set to list for each model
+    #     for model_name in deletions:
+    #         deletions[model_name] = deletions[model_name]
+
+    #     for model_name in grants:
+    #         grants[model_name] = list(grants[model_name].values())
+
+    #     # for model_name in revocation_parents:
+    #     #     revocation_parents[model_name] = list(revocation_parents[model_name].values())
+
+    #     response_data = {
+    #         "updates": dict(updates),
+    #         "deletions": dict(deletions),
+    #         "grants": dict(grants),
+    #         "revocations": dict(revocations),
+    #         "revocation_parents": revocations_parents_organizer.get_organized_data(),
+    #         "date": last_log_date
+    #         }
+
+    #     # print(f'sync GET response. UP = {len(updates)} / DEL = {len(deletions)} / G = {len(grants)} / R = {len(revocations)}')
+    #     # print(f'sync GET response. response = {response_data}')
+    #     return Response(response_data, status=status.HTTP_200_OK)
+
+## GET
+
     def get(self, request, *args, **kwargs):
         last_update_str = request.query_params.get('last_update')
         device_id = request.query_params.get('device_id')
 
-        # print(f'last_update_str = {last_update_str}')
         decoded_last_update = unquote(last_update_str)  # Decodes %2B into +
 
-        # print(f'last_update_str = {last_update_str}')
-        # print(f'decoded_last_update = {decoded_last_update}')
         if not decoded_last_update:
             return Response({"error": "last_update parameter is required"}, status=status.HTTP_400_BAD_REQUEST)
         try:
             last_update = timezone.datetime.fromisoformat(decoded_last_update)
         except ValueError:
-            return Response({"error": f"Invalid date format for last_update: {decoded_last_update}"}, status=status.HTTP_400_BAD_REQUEST)
-
-        # print(f'/data GET: {last_update}')
+            return Response({"error": f"Invalid date format for last_update: {decoded_last_update}"},
+                            status=status.HTTP_400_BAD_REQUEST)
 
         logs = self.query_model_logs(last_update, request.user, device_id)
         print(f'>>> log count = {len(logs)}')
 
+        # First pass: Process logs to collect basic operations
+        updates, deletions, grant_instances, revoke_info, last_log_date = self.process_logs(logs)
+
+        # Second pass: Process hierarchies for grants and revocations
+        grants = self.process_grants(grant_instances)
+        revocations_parents_organizer = self.process_revocations(revoke_info)
+
+        response_data = {
+            "updates": dict(updates),
+            "deletions": dict(deletions),
+            "grants": dict(grants),
+            "revocations": dict(revocations_parents_organizer[0]),
+            "revocation_parents": revocations_parents_organizer[1].get_organized_data(),
+            "date": last_log_date
+        }
+
+        return Response(response_data, status=status.HTTP_200_OK)
+
+    def process_logs(self, logs):
+        """Process logs to collect basic operations and handle grant/revoke efficiently."""
         updates = defaultdict(dict)
         deletions = defaultdict(list)
-        grants = defaultdict(dict)
-        revocations = defaultdict(list)
-        revocations_parents_organizer = HierarchyOrganizer()
-
-        # revocated_parents = defaultdict(dict)
+        grant_instances = defaultdict(dict)  # {model_name: {model_id: instance}}
+        revoke_info = defaultdict(list)  # {model_name: [{model_id, store_id}]}
 
         last_log_date = None
-        for log in logs:
-
-            # log.retrieved()
-            # log.save()
 
-            # print(f'log date = {log.date}')
+        for log in logs:
             last_log_date = log.date
             try:
                 if log.operation in ['POST', 'PUT']:
@@ -226,28 +333,28 @@ class SynchronizationApi(HierarchyApiView):
                 elif log.operation == 'DELETE':
                     deletions[log.model_name].append({'model_id': log.model_id, 'store_id': log.store_id})
                 elif log.operation == 'GRANT_ACCESS':
+                    # Remove any existing revocations for this model_id
+                    self._remove_revocation(revoke_info, log.model_name, log.model_id)
 
-                    model = model_registry.get_model(log.model_name)
-                    instance = model.objects.get(id=log.model_id)
-                    serializer = get_serializer(instance, log.model_name)
-
-                    grants[log.model_name][log.model_id] = serializer.data
-                    self.add_children_recursively(instance, grants)
-                    self.add_parents_recursively(instance, grants)
+                    # Add to grant instances if not already there
+                    if log.model_id not in grant_instances[log.model_name]:
+                        model = model_registry.get_model(log.model_name)
+                        try:
+                            instance = model.objects.get(id=log.model_id)
+                            grant_instances[log.model_name][log.model_id] = instance
+                        except model.DoesNotExist:
+                            pass
                 elif log.operation == 'REVOKE_ACCESS':
                     print(f'revoke access {log.model_id} - {log.store_id}')
-                    revocations[log.model_name].append({
+
+                    # Remove any existing grants for this model_id
+                    self._remove_grant(grant_instances, log.model_name, log.model_id)
+
+                    # Add to revocations
+                    revoke_info[log.model_name].append({
                         'model_id': log.model_id,
                         'store_id': log.store_id
                     })
-
-                    # Get the model instance and add its parents to hierarchy
-                    model = model_registry.get_model(log.model_name)
-                    try:
-                        instance = model.objects.get(id=log.model_id)
-                        self.add_parents_with_hierarchy_organizer(instance, revocations_parents_organizer)
-                    except model.DoesNotExist:
-                        pass
             except ObjectDoesNotExist:
                 pass
 
@@ -255,34 +362,72 @@ class SynchronizationApi(HierarchyApiView):
         for model_name in updates:
             updates[model_name] = list(updates[model_name].values())
 
-        # Convert deletions set to list for each model
-        for model_name in deletions:
-            deletions[model_name] = deletions[model_name]
+        return updates, deletions, grant_instances, revoke_info, last_log_date
+
+    def _remove_revocation(self, revoke_info, model_name, model_id):
+        """Remove any revocation entries for the specified model and ID."""
+        if model_name in revoke_info:
+            revoke_info[model_name] = [
+                r for r in revoke_info[model_name]
+                if r['model_id'] != model_id
+            ]
+            # Clean up empty lists
+            if not revoke_info[model_name]:
+                del revoke_info[model_name]
+
+    def _remove_grant(self, grant_instances, model_name, model_id):
+        """Remove any grant entries for the specified model and ID."""
+        if model_name in grant_instances and model_id in grant_instances[model_name]:
+            del grant_instances[model_name][model_id]
+            # Clean up empty dictionaries
+            if not grant_instances[model_name]:
+                del grant_instances[model_name]
+
+    def process_grants(self, grant_instances):
+        """Process grants and their hierarchies."""
+        grants = defaultdict(dict)
+
+        # Process each grant instance
+        for model_name, instances in grant_instances.items():
+            for model_id, instance in instances.items():
+                serializer = get_serializer(instance, model_name)
+                grants[model_name][model_id] = serializer.data
+
+                # Add hierarchies only once per instance
+                self.add_children_recursively(instance, grants)
+                self.add_parents_recursively(instance, grants)
 
+        # Convert to lists
         for model_name in grants:
             grants[model_name] = list(grants[model_name].values())
 
-        # for model_name in revocation_parents:
-        #     revocation_parents[model_name] = list(revocation_parents[model_name].values())
+        return grants
 
-        response_data = {
-            "updates": dict(updates),
-            "deletions": dict(deletions),
-            "grants": dict(grants),
-            "revocations": dict(revocations),
-            "revocation_parents": revocations_parents_organizer.get_organized_data(),
-            "date": last_log_date
-            }
+    def process_revocations(self, revoke_info):
+        """Process revocations and their hierarchies."""
+        revocations = defaultdict(list)
+        revocations_parents_organizer = HierarchyOrganizer()
 
-        # print(f'sync GET response. UP = {len(updates)} / DEL = {len(deletions)} / G = {len(grants)} / R = {len(revocations)}')
-        # print(f'sync GET response. response = {response_data}')
-        return Response(response_data, status=status.HTTP_200_OK)
+        # First, collect all revocations
+        for model_name, items in revoke_info.items():
+            revocations[model_name].extend(items)
+
+            # Process parent hierarchies for each revoked item
+            model = model_registry.get_model(model_name)
+            for item in items:
+                try:
+                    instance = model.objects.get(id=item['model_id'])
+                    self.add_parents_with_hierarchy_organizer(instance, revocations_parents_organizer)
+                except model.DoesNotExist:
+                    pass
+
+        return revocations, revocations_parents_organizer
 
     def query_model_logs(self, last_update, user, device_id):
         log_query = Q(date__gt=last_update, user=user)
         if device_id:
             log_query &= ~Q(device_id=device_id) # exclude query
-        return ModelLog.objects.filter(log_query)
+        return ModelLog.objects.filter(log_query).order_by('date')
 
 class UserDataAccessApi(HierarchyApiView):
     permission_classes = [IsAuthenticated]

tournaments/models/draw_log.py

@@ -17,10 +17,5 @@ class DrawLog(TournamentSubModel):
     def __str__(self):
         return f'{self.draw_date}'
 
-    def save(self, *args, **kwargs):
-        if self.tournament:
-            self.store_id = str(self.tournament.id)
-        super().save(*args, **kwargs)
-
-    def get_tournament_id(self):
-        return self.tournament.id
+    def get_tournament(self): # mandatory method for TournamentSubModel
+        return self.tournament