diff --git a/tournaments/admin_utils.py b/tournaments/admin_utils.py index 3ce3ffe..d3f521d 100644 --- a/tournaments/admin_utils.py +++ b/tournaments/admin_utils.py @@ -384,50 +384,85 @@ def download_french_padel_rankings(request): successful_calls = 0 failed_calls = 0 failed_tranches_list = [] + retry_stats = {} # Track retry attempts per tranche print(f"Starting to fetch tranches {start_tranche} to {end_tranche} ({total_tranches} total)...") - for tranche in range(start_tranche, end_tranche + 1): - try: - payload = { - "pratique": "padel", - "sexe": sexe, - "tranche": tranche - } + def fetch_tranche_with_retry(tranche, max_retries=10): + """ + Fetch a single tranche with retry logic + Returns: (success, players_data, retry_count) + """ + payload = { + "pratique": "padel", + "sexe": sexe, + "tranche": tranche + } - response = requests.post(url, json=payload, headers=headers, timeout=30) + for attempt in range(max_retries + 1): # +1 for initial attempt + try: + response = requests.post(url, json=payload, headers=headers, timeout=30) - if response.status_code == 200: - json_data = response.json() + if response.status_code == 200: + json_data = response.json() - if 'joueurs' in json_data and json_data['joueurs']: - # Add metadata to each player for enrichment tracking - for player in json_data['joueurs']: - player['source_tranche'] = tranche - player['license_lookup_status'] = 'not_attempted' - player['license_data'] = None + if 'joueurs' in json_data and json_data['joueurs']: + # Add metadata to each player for enrichment tracking + for player in json_data['joueurs']: + player['source_tranche'] = tranche + player['license_lookup_status'] = 'not_attempted' + player['license_data'] = None - all_players.extend(json_data['joueurs']) - successful_calls += 1 - print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players (Total: {len(all_players)})") + if attempt > 0: + print(f"Tranche {tranche}: SUCCESS after {attempt} retries - Found {len(json_data['joueurs'])} players") + else: + print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players") + + return True, json_data['joueurs'], attempt + else: + if attempt > 0: + print(f"Tranche {tranche}: SUCCESS after {attempt} retries - No players found") + else: + print(f"Tranche {tranche}: No players found") + return True, [], attempt else: - print(f"Tranche {tranche}: No players found") + if attempt < max_retries: + print(f"Tranche {tranche}: HTTP {response.status_code} - Retry {attempt + 1}/{max_retries}") + time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds + else: + print(f"Tranche {tranche}: FAILED after {max_retries} retries - HTTP {response.status_code}") - else: - failed_calls += 1 - failed_tranches_list.append(tranche) # Add this line - print(f"Tranche {tranche}: HTTP {response.status_code}") + except requests.exceptions.RequestException as e: + if attempt < max_retries: + print(f"Tranche {tranche}: Network error - {str(e)} - Retry {attempt + 1}/{max_retries}") + time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds + else: + print(f"Tranche {tranche}: FAILED after {max_retries} retries - Network error: {str(e)}") - except requests.exceptions.RequestException as e: - failed_calls += 1 - failed_tranches_list.append(tranche) # Add this line - print(f"Tranche {tranche}: Network error - {str(e)}") + except Exception as e: + if attempt < max_retries: + print(f"Tranche {tranche}: Unexpected error - {str(e)} - Retry {attempt + 1}/{max_retries}") + time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds + else: + print(f"Tranche {tranche}: FAILED after {max_retries} retries - Unexpected error: {str(e)}") + + return False, [], max_retries + + # Process all tranches with retry logic + for tranche in range(start_tranche, end_tranche + 1): + success, players_data, retry_count = fetch_tranche_with_retry(tranche) - except Exception as e: + if success: + all_players.extend(players_data) + successful_calls += 1 + if retry_count > 0: + retry_stats[tranche] = retry_count + else: failed_calls += 1 - failed_tranches_list.append(tranche) # Add this line - print(f"Tranche {tranche}: Unexpected error - {str(e)}") + failed_tranches_list.append(tranche) + retry_stats[tranche] = retry_count + # Progress update and small delay if tranche % 10 == 0: time.sleep(0.1) current_progress = tranche - start_tranche + 1 @@ -435,8 +470,29 @@ def download_french_padel_rankings(request): print(f"Completed! Total players found: {len(all_players)}") print(f"Successful calls: {successful_calls}, Failed calls: {failed_calls}") + + # Enhanced retry statistics logging + retry_summary = {} + tranches_with_retries = [t for t, c in retry_stats.items() if c > 0 and t not in failed_tranches_list] + if tranches_with_retries: + print(f"Tranches that required retries: {len(tranches_with_retries)}") + for tranche in sorted(tranches_with_retries): + retry_count = retry_stats[tranche] + print(f" Tranche {tranche}: {retry_count} retries") + if retry_count not in retry_summary: + retry_summary[retry_count] = 0 + retry_summary[retry_count] += 1 + + print("Retry distribution:") + for retry_count in sorted(retry_summary.keys()): + print(f" {retry_summary[retry_count]} tranches needed {retry_count} retries") + else: + print("No retries were needed!") + if failed_tranches_list: print(f"Failed tranches: {failed_tranches_list}") + failed_retry_counts = [retry_stats.get(t, 0) for t in failed_tranches_list] + print(f"All failed tranches attempted maximum retries (10)") else: print("No failed tranches - all requests successful!") @@ -447,10 +503,16 @@ def download_french_padel_rankings(request): "total_players": len(all_players), "successful_tranches": successful_calls, "failed_tranches": failed_calls, - "failed_tranches_list": failed_tranches_list, # Add this line + "failed_tranches_list": failed_tranches_list, "total_tranches_requested": total_tranches, "tranche_range": f"{start_tranche}-{end_tranche}", "download_timestamp": datetime.now().isoformat(), + "retry_statistics": { + "tranches_with_retries": len(tranches_with_retries), + "retry_stats_per_tranche": retry_stats, + "retry_distribution": retry_summary, + "max_retries_attempted": 10 + }, "last_enrichment_update": None, "enrichment_progress": { "players_with_licenses": 0, diff --git a/tournaments/management/commands/analyze_rankings.py b/tournaments/management/commands/analyze_rankings.py index 7dd4949..9ce09ba 100644 --- a/tournaments/management/commands/analyze_rankings.py +++ b/tournaments/management/commands/analyze_rankings.py @@ -56,15 +56,16 @@ class Command(BaseCommand): # Generate statistics if players: # self.generate_statistics(players, options) - - # Find anonymous players if requested - if options['find_anonymous']: - if options['auto_match']: - # Iterative approach: keep matching until no more changes can be made - self.iterative_match_anonymous_players(file_path, rankings_dir, options) - else: - # Single pass analysis without making changes - self.find_anonymous_players(players, metadata, rankings_dir, options, file_path) + self.iterative_match_anonymous_players(file_path, rankings_dir, options) + + # # Find anonymous players if requested + # if options['find_anonymous']: + # if options['auto_match']: + # # Iterative approach: keep matching until no more changes can be made + # self.iterative_match_anonymous_players(file_path, rankings_dir, options) + # else: + # # Single pass analysis without making changes + # self.find_anonymous_players(players, metadata, rankings_dir, options, file_path) def list_available_files(self, rankings_dir): """List all available ranking files"""