staxriver
/
padelclub_backend
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add retry logic and stats for padel rankings download

Browse Source 
The changes add a robust retry mechanism with exponential backoff for
downloading French padel rankings, along with detailed retry statistics
tracking and reporting. This improves reliability when fetching ranking
data in case of temporary network issues.
apikeys
Razmig Sarkissian 2 months ago
parent 8af498186c
commit e58ec43999
2 changed files with 104 additions and 41 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 126
      tournaments/admin_utils.py
  2. 19
      tournaments/management/commands/analyze_rankings.py

126
tournaments/admin_utils.py
Unescape View File

@ -384,50 +384,85 @@ def download_french_padel_rankings(request):
successful_calls = 0 successful_calls = 0
failed_calls = 0 failed_calls = 0
failed_tranches_list = [] failed_tranches_list = []
retry_stats = {} # Track retry attempts per tranche
print(f"Starting to fetch tranches {start_tranche} to {end_tranche} ({total_tranches} total)...") print(f"Starting to fetch tranches {start_tranche} to {end_tranche} ({total_tranches} total)...")
for tranche in range(start_tranche, end_tranche + 1): def fetch_tranche_with_retry(tranche, max_retries=10):
try: """
payload = { Fetch a single tranche with retry logic
"pratique": "padel", Returns: (success, players_data, retry_count)
"sexe": sexe, """
"tranche": tranche payload = {
} "pratique": "padel",
"sexe": sexe,
"tranche": tranche
}
response = requests.post(url, json=payload, headers=headers, timeout=30) for attempt in range(max_retries + 1): # +1 for initial attempt
try:
response = requests.post(url, json=payload, headers=headers, timeout=30)
if response.status_code == 200: if response.status_code == 200:
json_data = response.json() json_data = response.json()
if 'joueurs' in json_data and json_data['joueurs']: if 'joueurs' in json_data and json_data['joueurs']:
# Add metadata to each player for enrichment tracking # Add metadata to each player for enrichment tracking
for player in json_data['joueurs']: for player in json_data['joueurs']:
player['source_tranche'] = tranche player['source_tranche'] = tranche
player['license_lookup_status'] = 'not_attempted' player['license_lookup_status'] = 'not_attempted'
player['license_data'] = None player['license_data'] = None
all_players.extend(json_data['joueurs']) if attempt > 0:
successful_calls += 1 print(f"Tranche {tranche}: SUCCESS after {attempt} retries - Found {len(json_data['joueurs'])} players")
print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players (Total: {len(all_players)})") else:
print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players")
return True, json_data['joueurs'], attempt
else:
if attempt > 0:
print(f"Tranche {tranche}: SUCCESS after {attempt} retries - No players found")
else:
print(f"Tranche {tranche}: No players found")
return True, [], attempt
else: else:
print(f"Tranche {tranche}: No players found") if attempt < max_retries:
print(f"Tranche {tranche}: HTTP {response.status_code} - Retry {attempt + 1}/{max_retries}")
time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds
else:
print(f"Tranche {tranche}: FAILED after {max_retries} retries - HTTP {response.status_code}")
else: except requests.exceptions.RequestException as e:
failed_calls += 1 if attempt < max_retries:
failed_tranches_list.append(tranche) # Add this line print(f"Tranche {tranche}: Network error - {str(e)} - Retry {attempt + 1}/{max_retries}")
print(f"Tranche {tranche}: HTTP {response.status_code}") time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds
else:
print(f"Tranche {tranche}: FAILED after {max_retries} retries - Network error: {str(e)}")
except requests.exceptions.RequestException as e: except Exception as e:
failed_calls += 1 if attempt < max_retries:
failed_tranches_list.append(tranche) # Add this line print(f"Tranche {tranche}: Unexpected error - {str(e)} - Retry {attempt + 1}/{max_retries}")
print(f"Tranche {tranche}: Network error - {str(e)}") time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds
else:
print(f"Tranche {tranche}: FAILED after {max_retries} retries - Unexpected error: {str(e)}")
return False, [], max_retries
# Process all tranches with retry logic
for tranche in range(start_tranche, end_tranche + 1):
success, players_data, retry_count = fetch_tranche_with_retry(tranche)
except Exception as e: if success:
all_players.extend(players_data)
successful_calls += 1
if retry_count > 0:
retry_stats[tranche] = retry_count
else:
failed_calls += 1 failed_calls += 1
failed_tranches_list.append(tranche) # Add this line failed_tranches_list.append(tranche)
print(f"Tranche {tranche}: Unexpected error - {str(e)}") retry_stats[tranche] = retry_count
# Progress update and small delay
if tranche % 10 == 0: if tranche % 10 == 0:
time.sleep(0.1) time.sleep(0.1)
current_progress = tranche - start_tranche + 1 current_progress = tranche - start_tranche + 1
@ -435,8 +470,29 @@ def download_french_padel_rankings(request):
print(f"Completed! Total players found: {len(all_players)}") print(f"Completed! Total players found: {len(all_players)}")
print(f"Successful calls: {successful_calls}, Failed calls: {failed_calls}") print(f"Successful calls: {successful_calls}, Failed calls: {failed_calls}")
# Enhanced retry statistics logging
retry_summary = {}
tranches_with_retries = [t for t, c in retry_stats.items() if c > 0 and t not in failed_tranches_list]
if tranches_with_retries:
print(f"Tranches that required retries: {len(tranches_with_retries)}")
for tranche in sorted(tranches_with_retries):
retry_count = retry_stats[tranche]
print(f" Tranche {tranche}: {retry_count} retries")
if retry_count not in retry_summary:
retry_summary[retry_count] = 0
retry_summary[retry_count] += 1
print("Retry distribution:")
for retry_count in sorted(retry_summary.keys()):
print(f" {retry_summary[retry_count]} tranches needed {retry_count} retries")
else:
print("No retries were needed!")
if failed_tranches_list: if failed_tranches_list:
print(f"Failed tranches: {failed_tranches_list}") print(f"Failed tranches: {failed_tranches_list}")
failed_retry_counts = [retry_stats.get(t, 0) for t in failed_tranches_list]
print(f"All failed tranches attempted maximum retries (10)")
else: else:
print("No failed tranches - all requests successful!") print("No failed tranches - all requests successful!")
@ -447,10 +503,16 @@ def download_french_padel_rankings(request):
"total_players": len(all_players), "total_players": len(all_players),
"successful_tranches": successful_calls, "successful_tranches": successful_calls,
"failed_tranches": failed_calls, "failed_tranches": failed_calls,
"failed_tranches_list": failed_tranches_list, # Add this line "failed_tranches_list": failed_tranches_list,
"total_tranches_requested": total_tranches, "total_tranches_requested": total_tranches,
"tranche_range": f"{start_tranche}-{end_tranche}", "tranche_range": f"{start_tranche}-{end_tranche}",
"download_timestamp": datetime.now().isoformat(), "download_timestamp": datetime.now().isoformat(),
"retry_statistics": {
"tranches_with_retries": len(tranches_with_retries),
"retry_stats_per_tranche": retry_stats,
"retry_distribution": retry_summary,
"max_retries_attempted": 10
},
"last_enrichment_update": None, "last_enrichment_update": None,
"enrichment_progress": { "enrichment_progress": {
"players_with_licenses": 0, "players_with_licenses": 0,

19
tournaments/management/commands/analyze_rankings.py
Unescape View File

@ -56,15 +56,16 @@ class Command(BaseCommand):
# Generate statistics # Generate statistics
if players: if players:
# self.generate_statistics(players, options) # self.generate_statistics(players, options)
self.iterative_match_anonymous_players(file_path, rankings_dir, options)
# Find anonymous players if requested
if options['find_anonymous']: # # Find anonymous players if requested
if options['auto_match']: # if options['find_anonymous']:
# Iterative approach: keep matching until no more changes can be made # if options['auto_match']:
self.iterative_match_anonymous_players(file_path, rankings_dir, options) # # Iterative approach: keep matching until no more changes can be made
else: # self.iterative_match_anonymous_players(file_path, rankings_dir, options)
# Single pass analysis without making changes # else:
self.find_anonymous_players(players, metadata, rankings_dir, options, file_path) # # Single pass analysis without making changes
# self.find_anonymous_players(players, metadata, rankings_dir, options, file_path)
def list_available_files(self, rankings_dir): def list_available_files(self, rankings_dir):
"""List all available ranking files""" """List all available ranking files"""

Write Preview
Loading…
Cancel
Save