From e58ec4399996144c4e8ecdbfbb05e3616a86dd2b Mon Sep 17 00:00:00 2001 From: Razmig Sarkissian Date: Thu, 4 Sep 2025 08:45:44 +0200 Subject: [PATCH] Add retry logic and stats for padel rankings download The changes add a robust retry mechanism with exponential backoff for downloading French padel rankings, along with detailed retry statistics tracking and reporting. This improves reliability when fetching ranking data in case of temporary network issues. --- tournaments/admin_utils.py | 126 +++++++++++++----- .../management/commands/analyze_rankings.py | 19 +-- 2 files changed, 104 insertions(+), 41 deletions(-) diff --git a/tournaments/admin_utils.py b/tournaments/admin_utils.py index 3ce3ffe..d3f521d 100644 --- a/tournaments/admin_utils.py +++ b/tournaments/admin_utils.py @@ -384,50 +384,85 @@ def download_french_padel_rankings(request): successful_calls = 0 failed_calls = 0 failed_tranches_list = [] + retry_stats = {} # Track retry attempts per tranche print(f"Starting to fetch tranches {start_tranche} to {end_tranche} ({total_tranches} total)...") - for tranche in range(start_tranche, end_tranche + 1): - try: - payload = { - "pratique": "padel", - "sexe": sexe, - "tranche": tranche - } + def fetch_tranche_with_retry(tranche, max_retries=10): + """ + Fetch a single tranche with retry logic + Returns: (success, players_data, retry_count) + """ + payload = { + "pratique": "padel", + "sexe": sexe, + "tranche": tranche + } - response = requests.post(url, json=payload, headers=headers, timeout=30) + for attempt in range(max_retries + 1): # +1 for initial attempt + try: + response = requests.post(url, json=payload, headers=headers, timeout=30) - if response.status_code == 200: - json_data = response.json() + if response.status_code == 200: + json_data = response.json() - if 'joueurs' in json_data and json_data['joueurs']: - # Add metadata to each player for enrichment tracking - for player in json_data['joueurs']: - player['source_tranche'] = tranche - player['license_lookup_status'] = 'not_attempted' - player['license_data'] = None + if 'joueurs' in json_data and json_data['joueurs']: + # Add metadata to each player for enrichment tracking + for player in json_data['joueurs']: + player['source_tranche'] = tranche + player['license_lookup_status'] = 'not_attempted' + player['license_data'] = None - all_players.extend(json_data['joueurs']) - successful_calls += 1 - print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players (Total: {len(all_players)})") + if attempt > 0: + print(f"Tranche {tranche}: SUCCESS after {attempt} retries - Found {len(json_data['joueurs'])} players") + else: + print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players") + + return True, json_data['joueurs'], attempt + else: + if attempt > 0: + print(f"Tranche {tranche}: SUCCESS after {attempt} retries - No players found") + else: + print(f"Tranche {tranche}: No players found") + return True, [], attempt else: - print(f"Tranche {tranche}: No players found") + if attempt < max_retries: + print(f"Tranche {tranche}: HTTP {response.status_code} - Retry {attempt + 1}/{max_retries}") + time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds + else: + print(f"Tranche {tranche}: FAILED after {max_retries} retries - HTTP {response.status_code}") - else: - failed_calls += 1 - failed_tranches_list.append(tranche) # Add this line - print(f"Tranche {tranche}: HTTP {response.status_code}") + except requests.exceptions.RequestException as e: + if attempt < max_retries: + print(f"Tranche {tranche}: Network error - {str(e)} - Retry {attempt + 1}/{max_retries}") + time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds + else: + print(f"Tranche {tranche}: FAILED after {max_retries} retries - Network error: {str(e)}") - except requests.exceptions.RequestException as e: - failed_calls += 1 - failed_tranches_list.append(tranche) # Add this line - print(f"Tranche {tranche}: Network error - {str(e)}") + except Exception as e: + if attempt < max_retries: + print(f"Tranche {tranche}: Unexpected error - {str(e)} - Retry {attempt + 1}/{max_retries}") + time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds + else: + print(f"Tranche {tranche}: FAILED after {max_retries} retries - Unexpected error: {str(e)}") + + return False, [], max_retries + + # Process all tranches with retry logic + for tranche in range(start_tranche, end_tranche + 1): + success, players_data, retry_count = fetch_tranche_with_retry(tranche) - except Exception as e: + if success: + all_players.extend(players_data) + successful_calls += 1 + if retry_count > 0: + retry_stats[tranche] = retry_count + else: failed_calls += 1 - failed_tranches_list.append(tranche) # Add this line - print(f"Tranche {tranche}: Unexpected error - {str(e)}") + failed_tranches_list.append(tranche) + retry_stats[tranche] = retry_count + # Progress update and small delay if tranche % 10 == 0: time.sleep(0.1) current_progress = tranche - start_tranche + 1 @@ -435,8 +470,29 @@ def download_french_padel_rankings(request): print(f"Completed! Total players found: {len(all_players)}") print(f"Successful calls: {successful_calls}, Failed calls: {failed_calls}") + + # Enhanced retry statistics logging + retry_summary = {} + tranches_with_retries = [t for t, c in retry_stats.items() if c > 0 and t not in failed_tranches_list] + if tranches_with_retries: + print(f"Tranches that required retries: {len(tranches_with_retries)}") + for tranche in sorted(tranches_with_retries): + retry_count = retry_stats[tranche] + print(f" Tranche {tranche}: {retry_count} retries") + if retry_count not in retry_summary: + retry_summary[retry_count] = 0 + retry_summary[retry_count] += 1 + + print("Retry distribution:") + for retry_count in sorted(retry_summary.keys()): + print(f" {retry_summary[retry_count]} tranches needed {retry_count} retries") + else: + print("No retries were needed!") + if failed_tranches_list: print(f"Failed tranches: {failed_tranches_list}") + failed_retry_counts = [retry_stats.get(t, 0) for t in failed_tranches_list] + print(f"All failed tranches attempted maximum retries (10)") else: print("No failed tranches - all requests successful!") @@ -447,10 +503,16 @@ def download_french_padel_rankings(request): "total_players": len(all_players), "successful_tranches": successful_calls, "failed_tranches": failed_calls, - "failed_tranches_list": failed_tranches_list, # Add this line + "failed_tranches_list": failed_tranches_list, "total_tranches_requested": total_tranches, "tranche_range": f"{start_tranche}-{end_tranche}", "download_timestamp": datetime.now().isoformat(), + "retry_statistics": { + "tranches_with_retries": len(tranches_with_retries), + "retry_stats_per_tranche": retry_stats, + "retry_distribution": retry_summary, + "max_retries_attempted": 10 + }, "last_enrichment_update": None, "enrichment_progress": { "players_with_licenses": 0, diff --git a/tournaments/management/commands/analyze_rankings.py b/tournaments/management/commands/analyze_rankings.py index 7dd4949..9ce09ba 100644 --- a/tournaments/management/commands/analyze_rankings.py +++ b/tournaments/management/commands/analyze_rankings.py @@ -56,15 +56,16 @@ class Command(BaseCommand): # Generate statistics if players: # self.generate_statistics(players, options) - - # Find anonymous players if requested - if options['find_anonymous']: - if options['auto_match']: - # Iterative approach: keep matching until no more changes can be made - self.iterative_match_anonymous_players(file_path, rankings_dir, options) - else: - # Single pass analysis without making changes - self.find_anonymous_players(players, metadata, rankings_dir, options, file_path) + self.iterative_match_anonymous_players(file_path, rankings_dir, options) + + # # Find anonymous players if requested + # if options['find_anonymous']: + # if options['auto_match']: + # # Iterative approach: keep matching until no more changes can be made + # self.iterative_match_anonymous_players(file_path, rankings_dir, options) + # else: + # # Single pass analysis without making changes + # self.find_anonymous_players(players, metadata, rankings_dir, options, file_path) def list_available_files(self, rankings_dir): """List all available ranking files"""