from django.http import HttpResponse, JsonResponse
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib import messages
from django.shortcuts import redirect
from datetime import datetime
import requests
import json
import time
import unicodedata
import urllib.parse # Add this import
import os
from django.conf import settings
from django.template import Template, Context
from django.middleware.csrf import get_token
import concurrent.futures
from functools import partial
default_sexe = 'H'
default_id_homologation = "82546485"
default_session_id = "JSESSIONID=E3DE6A54D5367D48B0CFA970E09EB422; AWSALB=UlkEmLYVxfS3RNwiNeNygqdqjroNzOZF3D9k6nR+NP6YPG3r6JLIzOqtw3nV1aVKsyNMldzeFOmVy/V1OPf7LNVW/sckdD1EprkGtgqjX8N8DpihxhTGtTm+0sX1; AWSALBCORS=UlkEmLYVxfS3RNwiNeNygqdqjroNzOZF3D9k6nR+NP6YPG3r6JLIzOqtw3nV1aVKsyNMldzeFOmVy/V1OPf7LNVW/sckdD1EprkGtgqjX8N8DpihxhTGtTm+0sX1; datadome=K3v~wZc~sLs5C7D4p0OoS3jOXGpeDfai9vk~TDPw2mSFbxqpfjUcR68wvPaYXHYqXgAHOrFnrBGpoyNepJ6bXfncdSmYOUfMNPbAtvBBo67zZTxxSeogLiLu1U1_5Txo; TCID=; tc_cj_v2=%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKQOKSRRSNRRNQZZZ%5D777_rn_lh%5BfyfcheZZZ%2F%20%290%2BH%2C0%200%20G%24%2FH%29%20%2FZZZKQOLJNPJONLMPZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKQOLJNPJSSQMNZZZ%5D777_rn_lh%5BfyfcheZZZ%2F%20%290%2BH%2C0%200%20G%24%2FH%29%20%2FZZZKQOLJNPRRONPSZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKQOLJNQLKOMSOZZZ%5D777_rn_lh%5BfyfcheZZZ%2F%20%290%2BH%2C0%200%20G%24%2FH%29%20%2FZZZKQOLJPMNSNOJKZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKQOLJPMRPKRKMZZZ%5D777_rn_lh%5BfyfcheZZZ%2F%20%290%2BH%2C0%200%20G%24%2FH%29%20%2FZZZKQOLJQSONNLNQZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKQOLKMOPOJKSLZZZ%5D777_rn_lh%5BfyfcheZZZ%2F%20%290%2BH%2C0%200%20G%24%2FH%29%20%2FZZZKQONMQSSNRPKQZZZ%5D; tc_cj_v2_cmp=; tc_cj_v2_med=; xtan=-; xtant=1; pa_vid=%22mckhos3iasswydjm%22; _pcid=%7B%22browserId%22%3A%22mckhos3iasswydjm%22%2C%22_t%22%3A%22ms8wm9hs%7Cmckhos5s%22%7D; _pctx=%7Bu%7DN4IgrgzgpgThIC4B2YA2qA05owMoBcBDfSREQpAeyRCwgEt8oBJAE0RXSwH18yBbCAA4A7vwCcACwgAffgGMA1pMoQArPAC%2BQA; TCPID=125629554310878226394; xtvrn=$548419$"
def calculate_age_from_birth_date(birth_date_str):
"""
Calculate age from birth date string (format: DD/MM/YYYY)
Returns age as integer or None if parsing fails
"""
if not birth_date_str:
return None
try:
# Parse French date format DD/MM/YYYY
birth_date = datetime.strptime(birth_date_str, '%d/%m/%Y')
today = datetime.now()
age = today.year - birth_date.year - ((today.month, today.day) < (birth_date.month, birth_date.day))
return age
except (ValueError, TypeError):
return None
def find_best_license_match(license_results, player):
"""
Find the best matching license from multiple results using ageSportif comparison
Also filters out players without valid classement data
Args:
license_results: List of license data from API
player_age_sportif: Age from ranking data
Returns:
Tuple of (best_match, match_info)
"""
# Get player's age from ranking data for duplicate matching
player_age_sportif = player.get('ageSportif')
rank = player.get('classement')
lastname = player.get('nom')
firstname = player.get('prenom')
if not license_results:
return None, {"reason": "no_results"}
# First, filter out players without valid classement data
def has_valid_classement(license_data, rank):
"""Check if a license has valid classement data"""
classement = license_data.get('classement', {})
if not classement:
return False
# Check if any of the key classement fields have meaningful data
date_fr = classement.get('dateFr', '').strip()
rang = classement.get('rang')
points = classement.get('points')
date = classement.get('date')
# Consider it valid if at least one of these conditions is met:
# - dateFr is not empty
# - rang is not None
# - points is not None (and > 0)
# - date is not None
return (
rang is not None and rang == rank
)
# First, filter out players without valid classement data
def has_valid_name(license_data, firstname, lastname):
lk_firstname = license_data.get('prenom', '')
lk_lastname = license_data.get('nom', '')
if not lk_firstname and not lk_lastname:
return False
return (
lk_firstname == firstname and lk_lastname == lastname
)
# Filter license results to only include those with valid classement
valid_license_results = [
license_data for license_data in license_results
if has_valid_name(license_data, firstname, lastname)
if has_valid_classement(license_data, rank)
]
# If no valid results after filtering, return None
if not valid_license_results:
return None, {
"reason": "no_valid_classement",
"original_count": len(license_results),
"filtered_count": 0
}
# If only one valid result, return it
if len(valid_license_results) == 1:
return valid_license_results[0], {
"reason": "single_valid_result",
"original_count": len(license_results),
"filtered_count": 1,
"age_match": "n/a"
}
# If we don't have ageSportif from ranking, take the first valid match
if player_age_sportif is None:
return valid_license_results[0], {
"reason": "no_age_data_used_first_valid",
"original_count": len(license_results),
"filtered_count": len(valid_license_results),
"used_first_result": True
}
best_match = None
best_age_diff = float('inf')
match_details = []
best_match_count = 0
for i, license_data in enumerate(valid_license_results):
birth_date_fr = license_data.get('dateNaissanceFr')
calculated_age = calculate_age_from_birth_date(birth_date_fr)
match_detail = {
"index": i,
"dateNaissanceFr": birth_date_fr,
"calculated_age": calculated_age,
"player_age_sportif": player_age_sportif,
"age_difference": None,
"license": license_data.get('licence'),
"classement": license_data.get('classement', {})
}
if calculated_age is not None:
age_diff = abs(calculated_age - player_age_sportif)
match_detail["age_difference"] = age_diff
if age_diff < best_age_diff and best_age_diff > 1 and age_diff < 2:
best_age_diff = age_diff
best_match = license_data
best_match_count = 1
elif age_diff <= best_age_diff:
best_match_count += 1
match_details.append(match_detail)
# If no match found with valid age, use first valid result
if best_match is None:
match_info = {
"reason": "no_valid_ages_used_first_valid",
"original_count": len(license_results),
"filtered_count": len(valid_license_results),
"used_first_result": True,
"match_details": match_details
}
return valid_license_results[0], match_info
else:
if best_match_count == 1:
match_info = {
"reason": "age_matched",
"best_age_difference": best_age_diff,
"total_candidates": len(license_results),
"valid_candidates": len(valid_license_results),
"match_details": match_details
}
return best_match, match_info
else:
match_info = {
"reason": "multiple_matches",
"best_age_difference": best_age_diff,
"total_candidates": len(license_results),
"valid_candidates": len(valid_license_results),
"match_details": match_details
}
return None, match_info
@staff_member_required
def test_player_details_apis(request):
"""
Test different API endpoints to find player license information
"""
# Sample idCrm values from your data
test_ids = [5417299111, 9721526122]
results = {
"timestamp": datetime.now().isoformat(),
"test_results": []
}
for idCrm in test_ids:
player_results = {
"idCrm": idCrm,
"tests": []
}
# Test 1: Try player detail endpoint
try:
url = f"https://tenup.fft.fr/back/public/v1/joueurs/{idCrm}"
response = requests.get(url, timeout=10)
player_results["tests"].append({
"method": "GET",
"url": url,
"status_code": response.status_code,
"response_preview": response.text[:500] if response.status_code == 200 else response.text
})
except Exception as e:
player_results["tests"].append({
"method": "GET",
"url": url,
"error": str(e)
})
# Test 2: Try with different endpoint structure
try:
url = f"https://tenup.fft.fr/back/public/v1/players/{idCrm}"
response = requests.get(url, timeout=10)
player_results["tests"].append({
"method": "GET",
"url": url,
"status_code": response.status_code,
"response_preview": response.text[:500] if response.status_code == 200 else response.text
})
except Exception as e:
player_results["tests"].append({
"method": "GET",
"url": url,
"error": str(e)
})
# Test 3: Try POST with idCrm in body
try:
url = "https://tenup.fft.fr/back/public/v1/joueurs/detail"
payload = {"idCrm": idCrm}
response = requests.post(url, json=payload, headers={'Content-Type': 'application/json'}, timeout=10)
player_results["tests"].append({
"method": "POST",
"url": url,
"payload": payload,
"status_code": response.status_code,
"response_preview": response.text[:500] if response.status_code == 200 else response.text
})
except Exception as e:
player_results["tests"].append({
"method": "POST",
"url": url,
"payload": payload,
"error": str(e)
})
# Test 4: Try the classements endpoint with more parameters
try:
url = "https://tenup.fft.fr/back/public/v1/classements/recherche"
payload = {
"pratique": "padel",
"sexe": default_sexe,
"idCrm": idCrm
}
response = requests.post(url, json=payload, headers={'Content-Type': 'application/json'}, timeout=10)
player_results["tests"].append({
"method": "POST",
"url": url,
"payload": payload,
"status_code": response.status_code,
"response_preview": response.text[:500] if response.status_code == 200 else response.text
})
except Exception as e:
player_results["tests"].append({
"method": "POST",
"url": url,
"payload": payload,
"error": str(e)
})
results["test_results"].append(player_results)
time.sleep(0.5) # Small delay between tests
# Create filename with timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"fft_api_tests_{timestamp}.json"
# Return results as downloadable JSON
http_response = HttpResponse(
json.dumps(results, indent=2, ensure_ascii=False),
content_type='application/json; charset=utf-8'
)
http_response['Content-Disposition'] = f'attachment; filename="{filename}"'
return http_response
@staff_member_required
def explore_fft_api_endpoints(request):
"""
Try to discover FFT API endpoints
"""
base_url = "https://tenup.fft.fr/back/public/v1"
# Common API endpoint patterns to try
endpoints_to_test = [
"/joueurs",
"/players",
"/licences",
"/licenses",
"/classements",
"/rankings",
"/pratiques",
"/sports",
"/search",
"/recherche",
"/tournaments",
"/tournois"
]
results = {
"base_url": base_url,
"timestamp": datetime.now().isoformat(),
"endpoint_tests": []
}
for endpoint in endpoints_to_test:
try:
url = base_url + endpoint
response = requests.get(url, timeout=10)
results["endpoint_tests"].append({
"endpoint": endpoint,
"url": url,
"status_code": response.status_code,
"content_type": response.headers.get('content-type', ''),
"response_preview": response.text[:300] if len(response.text) < 1000 else response.text[:300] + "..."
})
except Exception as e:
results["endpoint_tests"].append({
"endpoint": endpoint,
"url": url,
"error": str(e)
})
time.sleep(0.2) # Small delay
# Create filename with timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"fft_api_discovery_{timestamp}.json"
# Return results as downloadable JSON
http_response = HttpResponse(
json.dumps(results, indent=2, ensure_ascii=False),
content_type='application/json; charset=utf-8'
)
http_response['Content-Disposition'] = f'attachment; filename="{filename}"'
return http_response
@staff_member_required
def download_french_padel_rankings(request):
"""
Download French padel rankings from FFT API and save locally for later enrichment
"""
if request.method == 'POST':
start_tranche = int(request.POST.get('start_tranche', 1001))
end_tranche = int(request.POST.get('end_tranche', 1222))
save_locally = request.POST.get('save_locally', 'true') == 'true'
sexe = request.POST.get('sexe', default_sexe)
try:
# API endpoint and parameters
url = "https://tenup.fft.fr/back/public/v1/classements/recherche"
headers = {
'Content-Type': 'application/json'
}
all_players = []
total_tranches = end_tranche - start_tranche + 1
successful_calls = 0
failed_calls = 0
failed_tranches_list = []
print(f"Starting to fetch tranches {start_tranche} to {end_tranche} ({total_tranches} total)...")
for tranche in range(start_tranche, end_tranche + 1):
try:
payload = {
"pratique": "padel",
"sexe": sexe,
"tranche": tranche
}
response = requests.post(url, json=payload, headers=headers, timeout=30)
if response.status_code == 200:
json_data = response.json()
if 'joueurs' in json_data and json_data['joueurs']:
# Add metadata to each player for enrichment tracking
for player in json_data['joueurs']:
player['source_tranche'] = tranche
player['license_lookup_status'] = 'not_attempted'
player['license_data'] = None
all_players.extend(json_data['joueurs'])
successful_calls += 1
print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players (Total: {len(all_players)})")
else:
print(f"Tranche {tranche}: No players found")
else:
failed_calls += 1
failed_tranches_list.append(tranche) # Add this line
print(f"Tranche {tranche}: HTTP {response.status_code}")
except requests.exceptions.RequestException as e:
failed_calls += 1
failed_tranches_list.append(tranche) # Add this line
print(f"Tranche {tranche}: Network error - {str(e)}")
except Exception as e:
failed_calls += 1
failed_tranches_list.append(tranche) # Add this line
print(f"Tranche {tranche}: Unexpected error - {str(e)}")
if tranche % 10 == 0:
time.sleep(0.1)
current_progress = tranche - start_tranche + 1
print(f"Progress: {current_progress}/{total_tranches} tranches processed...")
print(f"Completed! Total players found: {len(all_players)}")
print(f"Successful calls: {successful_calls}, Failed calls: {failed_calls}")
if failed_tranches_list:
print(f"Failed tranches: {failed_tranches_list}")
else:
print("No failed tranches - all requests successful!")
if all_players:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
final_data = {
"metadata": {
"total_players": len(all_players),
"successful_tranches": successful_calls,
"failed_tranches": failed_calls,
"failed_tranches_list": failed_tranches_list, # Add this line
"total_tranches_requested": total_tranches,
"tranche_range": f"{start_tranche}-{end_tranche}",
"download_timestamp": datetime.now().isoformat(),
"last_enrichment_update": None,
"enrichment_progress": {
"players_with_licenses": 0,
"players_without_licenses": len(all_players),
"last_processed_index": -1
},
"parameters": {
"pratique": "padel",
"sexe": sexe,
"tranche_start": start_tranche,
"tranche_end": end_tranche
}
},
"joueurs": all_players
}
# Save locally if requested
if save_locally:
rankings_dir = os.path.join(settings.BASE_DIR, 'data', 'rankings')
os.makedirs(rankings_dir, exist_ok=True)
filename = f"french_padel_rankings_{start_tranche}-{end_tranche}_{timestamp}.json"
local_file_path = os.path.join(rankings_dir, filename)
with open(local_file_path, 'w', encoding='utf-8') as f:
json.dump(final_data, f, indent=2, ensure_ascii=False)
print(f"Rankings saved locally to: {local_file_path}")
messages.success(request, f"Rankings saved locally to: {local_file_path}")
# Create download response
download_filename = f"french_padel_rankings_{start_tranche}-{end_tranche}_{timestamp}.json"
http_response = HttpResponse(
json.dumps(final_data, indent=2, ensure_ascii=False),
content_type='application/json; charset=utf-8'
)
http_response['Content-Disposition'] = f'attachment; filename="{download_filename}"'
return http_response
else:
messages.error(request, f"No players found in tranches {start_tranche}-{end_tranche}.")
except Exception as e:
messages.error(request, f"Unexpected error: {str(e)}")
csrf_token = get_token(request)
html_template = Template('''
Download French Padel Rankings - Debug Tools
๐ Download French Padel Rankings
๐ New Workflow:
1. Use this tool to download and save rankings locally
2. Then use "Enrich Rankings with Licenses" to add license data
3. Process is resumable - you can stop and restart enrichment
๐ก How to use: You'll need a valid sessionId (from Cookie header) and idHomologation (competition ID) from the FFT Beach-Padel website. Use browser dev tools to get these values.
๐ฌ API Research Tools
Tools to help discover and explore the FFT API endpoints.
'''
return HttpResponse(html_content)
@staff_member_required
def get_player_license_info(request):
"""
Get player license information using sessionId and idHomologation
"""
if request.method == 'POST':
session_id = request.POST.get('sessionId', '').strip()
id_homologation = request.POST.get('idHomologation', '').strip()
license_id = request.POST.get('licenseId', '').strip()
if not session_id or not id_homologation or not license_id:
messages.error(request, "sessionId, idHomologation, and licenseId are all required.")
return redirect(request.path)
try:
# Construct the URL (equivalent to Swift's URL construction)
url = f"https://beach-padel.app.fft.fr/beachja/rechercheJoueur/licencies?idHomologation={id_homologation}&numeroLicence={license_id}"
# Set up headers (equivalent to Swift's URLRequest headers)
headers = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Sec-Fetch-Site": "same-origin",
"Accept-Language": "fr-FR,fr;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Sec-Fetch-Mode": "cors",
"Host": "beach-padel.app.fft.fr",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
"Connection": "keep-alive",
"Referer": f"https://beach-padel.app.fft.fr/beachja/competitionFiche/inscrireEquipe?identifiantHomologation={id_homologation}",
"X-Requested-With": "XMLHttpRequest",
"Cookie": session_id
}
print(f"Making request to: {url}")
print(f"Headers: {headers}")
# Make the GET request (equivalent to Swift's URLSession.shared.data)
response = requests.get(url, headers=headers, timeout=30)
print(f"Response status: {response.status_code}")
print(f"Raw JSON response: {response.text}")
if response.status_code == 200:
try:
json_data = response.json()
# Create result structure
result = {
"request_info": {
"url": url,
"license_id": license_id,
"id_homologation": id_homologation,
"timestamp": datetime.now().isoformat()
},
"response_info": {
"status_code": response.status_code,
"headers": dict(response.headers),
"raw_response": response.text
},
"parsed_data": json_data
}
# Create filename with timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"player_license_info_{license_id}_{timestamp}.json"
# Return as downloadable JSON
http_response = HttpResponse(
json.dumps(result, indent=2, ensure_ascii=False),
content_type='application/json; charset=utf-8'
)
http_response['Content-Disposition'] = f'attachment; filename="{filename}"'
return http_response
except json.JSONDecodeError as e:
messages.error(request, f"Failed to parse JSON response: {str(e)}")
else:
messages.error(request, f"Request failed with status {response.status_code}: {response.text}")
except requests.exceptions.RequestException as e:
messages.error(request, f"Network error: {str(e)}")
except Exception as e:
messages.error(request, f"Unexpected error: {str(e)}")
csrf_token = get_token(request)
# Default values
default_license_id = "5186803"
html_template = Template('''
Player License Lookup - Debug Tools
๐ Player License Lookup
โ Default values loaded!
The form is pre-filled with your provided sessionId, idHomologation, and license ID.
You can modify any field if needed before submitting.
''')
context = Context({
'csrf_token': csrf_token,
'default_session_id': default_session_id,
'default_id_homologation': default_id_homologation,
'default_license_ids': default_license_ids,
'default_sexe': default_sexe
})
return HttpResponse(html_template.render(context))
@staff_member_required
def search_player_by_name(request):
"""
Search for players by nom and prenom
"""
if request.method == 'POST':
session_id = request.POST.get('sessionId', '').strip()
id_homologation = request.POST.get('idHomologation', '').strip()
nom = request.POST.get('nom', '')
prenom = request.POST.get('prenom', '')
sexe = request.POST.get('sexe', default_sexe)
if not session_id or not id_homologation:
messages.error(request, "sessionId and idHomologation are required.")
return redirect(request.path)
if not nom and not prenom:
messages.error(request, "At least nom or prenom is required.")
return redirect(request.path)
try:
# Construct the URL for name search
base_url = "https://beach-padel.app.fft.fr/beachja/rechercheJoueur/licencies"
params = {
"idHomologation": id_homologation
}
# Add name parameters if provided
if nom:
params["nom"] = nom
if prenom:
params["prenom"] = prenom
# Build query string
query_string = "&".join([f"{k}={v}" for k, v in params.items()])
url = f"{base_url}?{query_string}"
# Set up headers
headers = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Sec-Fetch-Site": "same-origin",
"Accept-Language": "fr-FR,fr;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Sec-Fetch-Mode": "cors",
"Host": "beach-padel.app.fft.fr",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
"Connection": "keep-alive",
"Referer": f"https://beach-padel.app.fft.fr/beachja/competitionFiche/inscrireEquipe?identifiantHomologation={id_homologation}",
"X-Requested-With": "XMLHttpRequest",
"Cookie": session_id
}
print(f"Making request to: {url}")
print(f"Headers: {headers}")
# Make the GET request
response = requests.get(url, headers=headers, timeout=30)
print(f"Response status: {response.status_code}")
print(f"Raw JSON response: {response.text}")
if response.status_code == 200:
try:
json_data = response.json()
# Create result structure
result = {
"search_info": {
"url": url,
"nom": nom,
"prenom": prenom,
"id_homologation": id_homologation,
"timestamp": datetime.now().isoformat()
},
"response_info": {
"status_code": response.status_code,
"headers": dict(response.headers),
"raw_response": response.text
},
"parsed_data": json_data
}
# Create filename with timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
search_term = f"{nom}_{prenom}".replace(" ", "_")
filename = f"player_search_{search_term}_{timestamp}.json"
# Return as downloadable JSON
http_response = HttpResponse(
json.dumps(result, indent=2, ensure_ascii=False),
content_type='application/json; charset=utf-8'
)
http_response['Content-Disposition'] = f'attachment; filename="{filename}"'
return http_response
except json.JSONDecodeError as e:
messages.error(request, f"Failed to parse JSON response: {str(e)}")
else:
messages.error(request, f"Request failed with status {response.status_code}: {response.text}")
except requests.exceptions.RequestException as e:
messages.error(request, f"Network error: {str(e)}")
except Exception as e:
messages.error(request, f"Unexpected error: {str(e)}")
csrf_token = get_token(request)
# Default values
html_template = Template('''
Search Player by Name - Debug Tools
๐ Search Player by Name
โ Default values loaded!
ID Homologation is pre-filled. Add your session ID and enter the player's name to search.
โน๏ธ How it works:
โข You can search by nom (last name), prenom (first name), or both
โข Partial names should work (e.g., "DUPON" might find "DUPONT")
โข Leave one field empty to search by the other
โข Results will include license numbers and detailed player information
๐ Example Searches
Search by last name only: nom = "MARTIN", prenom = empty
Search by first name only: nom = empty, prenom = "Jean"
Search by both: nom = "DUPONT", prenom = "Pierre"
Partial search: nom = "DUPON" (might find "DUPONT", "DUPONTEL", etc.)
''')
context = Context({
'csrf_token': csrf_token,
'default_session_id': default_session_id,
'default_id_homologation': default_id_homologation,
'default_sexe': default_sexe
})
return HttpResponse(html_template.render(context))
@staff_member_required
def enrich_rankings_with_licenses(request):
"""
Load a local rankings file and enrich players with license data (resumable)
Uses concurrent processing to speed up lookups for large files.
"""
if request.method == 'POST':
file_path = request.POST.get('file_path', '').strip()
session_id = request.POST.get('sessionId', default_session_id).strip()
id_homologation = request.POST.get('idHomologation', default_id_homologation).strip()
save_batch_size = int(request.POST.get('batch_size', 1000)) # How often to save progress
max_workers = int(request.POST.get('max_workers', 10)) # New parameter for controlling concurrency
if not file_path or not session_id or not id_homologation:
messages.error(request, "File path, session ID, and ID homologation are required.")
return redirect(request.path)
try:
# Load the rankings file
if not os.path.exists(file_path):
messages.error(request, f"File not found: {file_path}")
return redirect(request.path)
print(f"Loading rankings from: {file_path}")
with open(file_path, 'r', encoding='utf-8') as f:
data = json.load(f)
players = data.get('joueurs', [])
metadata = data.get('metadata', {})
# Extract sex from metadata (it's the same for all players in the ranking)
sexe = metadata.get('parameters', {}).get('sexe', default_sexe)
print(f"Using sex from metadata: {sexe}")
if not players:
messages.error(request, "No players found in the file.")
return redirect(request.path)
print(f"Loaded {len(players)} players from file")
# Define a worker function that will process a single player
def lookup_player_license(player_tuple, session_id, id_homologation, sexe):
original_index, player = player_tuple
# Extract and normalize names
raw_nom = player.get('nom')
raw_prenom = player.get('prenom')
if not raw_nom or not raw_prenom:
player['license_lookup_status'] = 'missing_name_data'
return player, False, None
# Keep original case and accents, just clean up any extra whitespace
nom = raw_nom
prenom = raw_prenom
# Setup license lookup headers
license_headers = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Sec-Fetch-Site": "same-origin",
"Accept-Language": "fr-FR,fr;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Sec-Fetch-Mode": "cors",
"Host": "beach-padel.app.fft.fr",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
"Connection": "keep-alive",
"Referer": f"https://beach-padel.app.fft.fr/beachja/competitionFiche/inscrireEquipe?identifiantHomologation={id_homologation}",
"X-Requested-With": "XMLHttpRequest",
"Cookie": session_id
}
def sanitize_for_latin1(text):
# Replace specific problematic characters
text = text.replace('\u2019', "'") # Replace right single quotation mark with regular apostrophe
# Add more replacements as needed
return text
try:
# Build license lookup URL with proper URL encoding
license_url = f"https://beach-padel.app.fft.fr/beachja/rechercheJoueur/licencies?idHomologation={id_homologation}&nom={urllib.parse.quote(sanitize_for_latin1(nom), encoding='latin-1')}&prenom={urllib.parse.quote(sanitize_for_latin1(prenom), encoding='latin-1')}&sexe={sexe}"
# Make license lookup request
license_response = requests.get(license_url, headers=license_headers, timeout=30)
license_response.encoding = 'utf-8' # Ensure consistent encoding
if license_response.status_code == 200:
try:
license_data = license_response.json()
if license_data and 'object' in license_data and 'listeJoueurs' in license_data['object']:
liste_joueurs = license_data['object']['listeJoueurs']
presence_doublon = license_data['object'].get('presenceDoublon', False)
if liste_joueurs:
# Find the best matching license using age comparison
license_info, match_info = find_best_license_match(liste_joueurs, player)
if license_info is None:
player['license_lookup_status'] = 'too_many_results'
player['presenceDoublon'] = presence_doublon
return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> Too many results"
# Add all license data to player
player['licence'] = license_info.get('licence')
player['codeClub'] = license_info.get('codeClub')
player['nomClub'] = license_info.get('nomClub')
# Track duplicates and matching info
player['presenceDoublon'] = presence_doublon
player['nombreResultatsLicence'] = len(liste_joueurs)
player['license_match_info'] = match_info
if len(liste_joueurs) > 1:
player['tousResultatsLicence'] = liste_joueurs
player['license_lookup_status'] = 'success'
# Enhanced logging with age matching info
doublon_status = ""
if presence_doublon:
if match_info["reason"] == "age_matched":
age_diff = match_info.get("best_age_difference", "?")
doublon_status = f" (DOUBLON - matched by age, diff: {age_diff})"
elif match_info["reason"] == "no_age_data":
doublon_status = " (DOUBLON - no age data, used first)"
else:
doublon_status = f" (DOUBLON - {match_info['reason']})"
return player, True, f"โ {nom} {prenom} -> {license_info.get('licence')}{doublon_status}"
else:
player['license_lookup_status'] = 'no_results'
player['presenceDoublon'] = presence_doublon
return player, False, f"Failed {nom} {prenom} {player['idCrm']} {presence_doublon} -> No results"
else:
player['license_lookup_status'] = 'no_data'
return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> No data"
except json.JSONDecodeError as json_err:
player['license_lookup_status'] = 'json_decode_error'
player['license_lookup_error'] = str(json_err)
player['raw_response'] = license_response.text
# Debug info for JSON decode errors
error_position = json_err.pos
return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> JSON decode error: {str(json_err)} at pos {error_position}"
else:
player['license_lookup_status'] = f'http_error_{license_response.status_code}'
return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> HTTP {license_response.status_code}"
except requests.exceptions.RequestException as e:
player['license_lookup_status'] = f'network_error: {type(e).__name__}'
player['license_lookup_error'] = str(e)
return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> Network error: {type(e).__name__}: {str(e)}"
# Find players that need license enrichment
players_needing_licenses = []
players_with_licenses = 0
for i, player in enumerate(players):
if player is None:
continue
if player.get('license_lookup_status') not in ['success']:
players_needing_licenses.append((i, player))
else:
players_with_licenses += 1
print(f"Players with licenses: {players_with_licenses}")
print(f"Players needing licenses: {len(players_needing_licenses)}")
if not players_needing_licenses:
messages.success(request, "All players already have license data!")
return redirect(request.path)
# Process players concurrently in batches
total_processed = 0
successful_lookups = 0
failed_lookups = 0
print(f"Starting concurrent enrichment of {len(players_needing_licenses)} players with {max_workers} workers")
# Prepare the partial function with fixed parameters
worker_fn = partial(lookup_player_license, session_id=session_id, id_homologation=id_homologation, sexe=sexe)
# Process in batches for saving progress, using the value from the form
for batch_start in range(0, len(players_needing_licenses), save_batch_size):
batch_end = min(batch_start + save_batch_size, len(players_needing_licenses))
current_batch = players_needing_licenses[batch_start:batch_end]
print(f"\n=== Processing batch {batch_start//save_batch_size + 1}: players {batch_start+1} to {batch_end} ===")
batch_results = []
# Process the current batch with concurrent workers
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_player = {executor.submit(worker_fn, player_tuple): player_tuple for player_tuple in current_batch}
for future in concurrent.futures.as_completed(future_to_player):
player_tuple = future_to_player[future]
original_index = player_tuple[0]
try:
updated_player, success, message = future.result()
players[original_index] = updated_player # Update the player in the main list
if success:
successful_lookups += 1
else:
failed_lookups += 1
total_processed += 1
# Print progress every 10 players
if total_processed % 10 == 0 or total_processed == len(players_needing_licenses):
print(f" Progress: {total_processed}/{len(players_needing_licenses)} ({(total_processed/len(players_needing_licenses)*100):.1f}%)")
if message:
batch_results.append(message)
except Exception as e:
print(f" ERROR processing player {original_index} ({player_tuple}'): {str(e)}")
failed_lookups += 1
total_processed += 1
# Print batch results
for msg in batch_results:
print(f" {msg}")
# Save progress after each batch
metadata['last_enrichment_update'] = datetime.now().isoformat()
metadata['enrichment_progress'] = {
'players_with_licenses': successful_lookups + players_with_licenses,
'players_without_licenses': len(players) - (successful_lookups + players_with_licenses),
'last_processed_index': batch_end - 1,
'total_processed_this_run': total_processed,
'successful_this_run': successful_lookups,
'failed_this_run': failed_lookups
}
# Save the updated file
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
print(f"Progress saved: {successful_lookups + players_with_licenses}/{len(players)} players have licenses")
print(f"\n=== ENRICHMENT COMPLETE ===")
print(f"Total processed this run: {total_processed}")
print(f"Successful lookups this run: {successful_lookups}")
print(f"Failed lookups this run: {failed_lookups}")
print(f"Total players with licenses: {successful_lookups + players_with_licenses}/{len(players)}")
messages.success(request, f"Enrichment complete! Processed {total_processed} players. {successful_lookups + players_with_licenses}/{len(players)} players now have license data.")
except Exception as e:
import traceback
error_traceback = traceback.format_exc()
print(f"CRITICAL ERROR: {type(e).__name__}: {str(e)}")
print(f"Traceback: {error_traceback}")
messages.error(request, f"Error during enrichment: {type(e).__name__}: {str(e)}")
return redirect(request.path)
# Show the form
from django.template import Template, Context
from django.middleware.csrf import get_token
csrf_token = get_token(request)
# Try to find existing ranking files
rankings_dir = os.path.join(settings.BASE_DIR, 'data', 'rankings')
existing_files = []
if os.path.exists(rankings_dir):
for filename in os.listdir(rankings_dir):
if filename.endswith('.json') and 'french_padel_rankings' in filename:
file_path = os.path.join(rankings_dir, filename)
existing_files.append({
'filename': filename,
'path': file_path,
'size': os.path.getsize(file_path),
'modified': datetime.fromtimestamp(os.path.getmtime(file_path)).strftime('%Y-%m-%d %H:%M:%S')
})
existing_files.sort(key=lambda x: x['modified'], reverse=True)
html_template = Template('''
Enrich Rankings with Licenses - Debug Tools
Enrich Rankings with Licenses
Optimized for large datasets: This tool now supports concurrent processing to speed up license lookups.
You can adjust the batch size and number of concurrent workers based on your server's capacity.