staxriver
/
padelclub_backend
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add retry logic and stats for padel rankings download

Browse Source 
The changes add a robust retry mechanism with exponential backoff for
downloading French padel rankings, along with detailed retry statistics
tracking and reporting. This improves reliability when fetching ranking
data in case of temporary network issues.
apikeys
Razmig Sarkissian 2 months ago
parent 8af498186c
commit e58ec43999
2 changed files with 104 additions and 41 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 92
      tournaments/admin_utils.py
  2. 17
      tournaments/management/commands/analyze_rankings.py

92
tournaments/admin_utils.py
Unescape View File

@ -384,17 +384,23 @@ def download_french_padel_rankings(request):
successful_calls = 0
failed_calls = 0
failed_tranches_list = []
retry_stats = {} # Track retry attempts per tranche
print(f"Starting to fetch tranches {start_tranche} to {end_tranche} ({total_tranches} total)...")
for tranche in range(start_tranche, end_tranche + 1):
try:
def fetch_tranche_with_retry(tranche, max_retries=10):
"""
Fetch a single tranche with retry logic
Returns: (success, players_data, retry_count)
"""
payload = {
"pratique": "padel",
"sexe": sexe,
"tranche": tranche
}
for attempt in range(max_retries + 1): # +1 for initial attempt
try:
response = requests.post(url, json=payload, headers=headers, timeout=30)
if response.status_code == 200:
@ -407,27 +413,56 @@ def download_french_padel_rankings(request):
player['license_lookup_status'] = 'not_attempted'
player['license_data'] = None
all_players.extend(json_data['joueurs'])
successful_calls += 1
print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players (Total: {len(all_players)})")
if attempt > 0:
print(f"Tranche {tranche}: SUCCESS after {attempt} retries - Found {len(json_data['joueurs'])} players")
else:
print(f"Tranche {tranche}: No players found")
print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players")
return True, json_data['joueurs'], attempt
else:
failed_calls += 1
failed_tranches_list.append(tranche) # Add this line
print(f"Tranche {tranche}: HTTP {response.status_code}")
if attempt > 0:
print(f"Tranche {tranche}: SUCCESS after {attempt} retries - No players found")
else:
print(f"Tranche {tranche}: No players found")
return True, [], attempt
else:
if attempt < max_retries:
print(f"Tranche {tranche}: HTTP {response.status_code} - Retry {attempt + 1}/{max_retries}")
time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds
else:
print(f"Tranche {tranche}: FAILED after {max_retries} retries - HTTP {response.status_code}")
except requests.exceptions.RequestException as e:
failed_calls += 1
failed_tranches_list.append(tranche) # Add this line
print(f"Tranche {tranche}: Network error - {str(e)}")
if attempt < max_retries:
print(f"Tranche {tranche}: Network error - {str(e)} - Retry {attempt + 1}/{max_retries}")
time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds
else:
print(f"Tranche {tranche}: FAILED after {max_retries} retries - Network error: {str(e)}")
except Exception as e:
if attempt < max_retries:
print(f"Tranche {tranche}: Unexpected error - {str(e)} - Retry {attempt + 1}/{max_retries}")
time.sleep(min(2 ** attempt, 10)) # Exponential backoff, max 10 seconds
else:
print(f"Tranche {tranche}: FAILED after {max_retries} retries - Unexpected error: {str(e)}")
return False, [], max_retries
# Process all tranches with retry logic
for tranche in range(start_tranche, end_tranche + 1):
success, players_data, retry_count = fetch_tranche_with_retry(tranche)
if success:
all_players.extend(players_data)
successful_calls += 1
if retry_count > 0:
retry_stats[tranche] = retry_count
else:
failed_calls += 1
failed_tranches_list.append(tranche) # Add this line
print(f"Tranche {tranche}: Unexpected error - {str(e)}")
failed_tranches_list.append(tranche)
retry_stats[tranche] = retry_count
# Progress update and small delay
if tranche % 10 == 0:
time.sleep(0.1)
current_progress = tranche - start_tranche + 1
@ -435,8 +470,29 @@ def download_french_padel_rankings(request):
print(f"Completed! Total players found: {len(all_players)}")
print(f"Successful calls: {successful_calls}, Failed calls: {failed_calls}")
# Enhanced retry statistics logging
retry_summary = {}
tranches_with_retries = [t for t, c in retry_stats.items() if c > 0 and t not in failed_tranches_list]
if tranches_with_retries:
print(f"Tranches that required retries: {len(tranches_with_retries)}")
for tranche in sorted(tranches_with_retries):
retry_count = retry_stats[tranche]
print(f" Tranche {tranche}: {retry_count} retries")
if retry_count not in retry_summary:
retry_summary[retry_count] = 0
retry_summary[retry_count] += 1
print("Retry distribution:")
for retry_count in sorted(retry_summary.keys()):
print(f" {retry_summary[retry_count]} tranches needed {retry_count} retries")
else:
print("No retries were needed!")
if failed_tranches_list:
print(f"Failed tranches: {failed_tranches_list}")
failed_retry_counts = [retry_stats.get(t, 0) for t in failed_tranches_list]
print(f"All failed tranches attempted maximum retries (10)")
else:
print("No failed tranches - all requests successful!")
@ -447,10 +503,16 @@ def download_french_padel_rankings(request):
"total_players": len(all_players),
"successful_tranches": successful_calls,
"failed_tranches": failed_calls,
"failed_tranches_list": failed_tranches_list, # Add this line
"failed_tranches_list": failed_tranches_list,
"total_tranches_requested": total_tranches,
"tranche_range": f"{start_tranche}-{end_tranche}",
"download_timestamp": datetime.now().isoformat(),
"retry_statistics": {
"tranches_with_retries": len(tranches_with_retries),
"retry_stats_per_tranche": retry_stats,
"retry_distribution": retry_summary,
"max_retries_attempted": 10
},
"last_enrichment_update": None,
"enrichment_progress": {
"players_with_licenses": 0,

17
tournaments/management/commands/analyze_rankings.py
Unescape View File

@ -56,15 +56,16 @@ class Command(BaseCommand):
# Generate statistics
if players:
# self.generate_statistics(players, options)
# Find anonymous players if requested
if options['find_anonymous']:
if options['auto_match']:
# Iterative approach: keep matching until no more changes can be made
self.iterative_match_anonymous_players(file_path, rankings_dir, options)
else:
# Single pass analysis without making changes
self.find_anonymous_players(players, metadata, rankings_dir, options, file_path)
# # Find anonymous players if requested
# if options['find_anonymous']:
# if options['auto_match']:
# # Iterative approach: keep matching until no more changes can be made
# self.iterative_match_anonymous_players(file_path, rankings_dir, options)
# else:
# # Single pass analysis without making changes
# self.find_anonymous_players(players, metadata, rankings_dir, options, file_path)
def list_available_files(self, rankings_dir):
"""List all available ranking files"""

Write Preview
Loading…
Cancel
Save