diff --git a/padelclub_backend/urls.py b/padelclub_backend/urls.py index abb883a..d56034b 100644 --- a/padelclub_backend/urls.py +++ b/padelclub_backend/urls.py @@ -17,6 +17,7 @@ from django.contrib import admin from django.urls import include, path from django.conf import settings from django.conf.urls.static import static +from tournaments.admin_utils import download_french_padel_rankings, debug_tools_page, test_player_details_apis, explore_fft_api_endpoints, get_player_license_info, bulk_license_lookup, search_player_by_name, enrich_rankings_with_licenses urlpatterns = [ @@ -24,6 +25,14 @@ urlpatterns = [ path('shop/', include('shop.urls')), # path("crm/", include("crm.urls")), path('roads/', include("api.urls")), + path('kingdom/debug/', debug_tools_page, name='debug_tools'), + path('kingdom/debug/enrich-rankings-with-licenses/', enrich_rankings_with_licenses, name='enrich_rankings_with_licenses'), + path('kingdom/debug/search-player-by-name/', search_player_by_name, name='search_player_by_name'), + path('kingdom/debug/download-french-padel-rankings/', download_french_padel_rankings, name='download_french_padel_rankings'), + path('kingdom/debug/test-player-apis/', test_player_details_apis, name='test_player_apis'), + path('kingdom/debug/player-license-lookup/', get_player_license_info, name='player_license_lookup'), + path('kingdom/debug/bulk-license-lookup/', bulk_license_lookup, name='bulk_license_lookup'), + path('kingdom/debug/explore-api-endpoints/', explore_fft_api_endpoints, name='explore_api_endpoints'), path('kingdom/', admin.site.urls), path('api-auth/', include('rest_framework.urls')), path('dj-auth/', include('django.contrib.auth.urls')), diff --git a/tournaments/admin_utils.py b/tournaments/admin_utils.py new file mode 100644 index 0000000..43e8675 --- /dev/null +++ b/tournaments/admin_utils.py @@ -0,0 +1,1739 @@ +from django.http import HttpResponse, JsonResponse +from django.contrib.admin.views.decorators import staff_member_required +from django.contrib import messages +from django.shortcuts import redirect +from datetime import datetime +import requests +import json +import time +import unicodedata +import urllib.parse # Add this import +import os +from django.conf import settings +from django.template import Template, Context +from django.middleware.csrf import get_token +import concurrent.futures +from functools import partial + +default_sexe = "H" +default_id_homologation = "82546485" +default_session_id = "JSESSIONID=CFE4A886CB92764066D1EC920EF9AB1C; AWSALB=c4OHU4Lw6YW6QRsoD1ktcfSgEExZZil/dDetMp3teMKtQ7RlA3VIM8ZHnJH8K3GYMoPu0M61xLjZo64rBNzDEO5tISKEYIX79baengXCKXnaqdqNqHJ7cSPeon+g; AWSALBCORS=c4OHU4Lw6YW6QRsoD1ktcfSgEExZZil/dDetMp3teMKtQ7RlA3VIM8ZHnJH8K3GYMoPu0M61xLjZo64rBNzDEO5tISKEYIX79baengXCKXnaqdqNqHJ7cSPeon+g; tc_cj_v2=m_iZZZ%22**%22%27%20ZZZKQNRSMQNLMRQLZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKQOKMQKQONJKOZZZ%5D777_rn_lh%5BfyfcheZZZ222H%2B%7B%7E%20%27-%20%21%20-%20%29%7D%20H%7D*%28ZZZKQOKMQKRLNNPMZZZ%5D777%5Ecl_%5Dny%5B%5D%5D_mmZZZZZZKQOKNLQOPMLSMZZZ%5D; tc_cj_v2_cmp=; tc_cj_v2_med=; incap_ses_2223_2712217=/I1fA56LxWI8KbyfPa/ZHpmpZGgAAAAAoadzHEsPdo9W59AkhfXcmQ==; xtan=-; xtant=1; pa_vid=%22mckhos3iasswydjm%22; datadome=oi7wKIO2uaUDCcpICiRO1_hEYcwyQWDVbXyNCSkAmr315~8pnPcuXWKfvNEEz~jKcoORIOThSRe~AxoRRrPLUsr0miWm7AdAcy~_3hABc1ZWfRt~SKGa_uhyqiE0Hzfj; _pcid=%7B%22browserId%22%3A%22mckhos3iasswydjm%22%2C%22_t%22%3A%22ms8wm9hs%7Cmckhos5s%22%7D; _pctx=%7Bu%7DN4IgrgzgpgThIC4B2YA2qA05owMoBcBDfSREQpAeyRCwgEt8oBJAE0RXSwH18yBbCAA4A7vwCcACwgAffgGMA1pMoQArPAC%2BQA; EA_SESSION_ID=E15E1DD5A23272A1A0CC3B8CEDF56B65; refresh_token=eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIzYjQ2ODk1ZC0zN2EzLTQzM2QtYmQ1My01N2QxZTM1YTI3NzkifQ.eyJleHAiOjE3NTY1NTM5MjgsImlhdCI6MTc1MTM3MjAwNCwianRpIjoiYzJiNzA3N2UtZmQ5MS00ZGM4LWI4ZDEtMzA2MDdkYjk5MTgxIiwiaXNzIjoiaHR0cHM6Ly9sb2dpbi5mZnQuZnIvcmVhbG1zL2Nvbm5lY3QiLCJhdWQiOiJodHRwczovL2xvZ2luLmZmdC5mci9yZWFsbXMvY29ubmVjdCIsInN1YiI6IjI3ZDQ5NzRjLTEwZWUtNDNlOC1iOTczLWUyMzc2MDM1ZTE0MSIsInR5cCI6IlJlZnJlc2giLCJhenAiOiJtZWEtc2l0ZSIsInNpZCI6IjM5NTZjMzZlLTczMWItNDJkNy1iNjI2LTE2MGViY2Y2YTY2ZiIsInNjb3BlIjoib3BlbmlkIHJvbGVzIHJlYWQ6bGljZW5jZSByZWFkOmlkZW50aXR5IGVtYWlsIHByb2ZpbGUifQ.e6v5vlen985vSFJhrgMQTTB3fzzsnwugPfXKoyib1QSIBZ9kC47h1cYwcpam0VmZ9vRD_y0hVC14jDvBR6d1dQ; user_login=10000984864; user_token=eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJRaTV3bWx2bTNuX2p1YW4tSTl1dHo3UGZRLU1tVVlvektwSExhbm9lTXI4In0.eyJleHAiOjE3NTEzNzIzMDQsImlhdCI6MTc1MTM3MjAwNCwianRpIjoiMzEzMGVhODUtNjFjNC00OGRjLWFlNGMtZTIwZmZhYTU3YTlhIiwiaXNzIjoiaHR0cHM6Ly9sb2dpbi5mZnQuZnIvcmVhbG1zL2Nvbm5lY3QiLCJhdWQiOlsiZmVkLWFwaSIsImFjY291bnQiXSwic3ViIjoiMjdkNDk3NGMtMTBlZS00M2U4LWI5NzMtZTIzNzYwMzVlMTQxIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoibWVhLXNpdGUiLCJzaWQiOiIzOTU2YzM2ZS03MzFiLTQyZDctYjYyNi0xNjBlYmNmNmE2NmYiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiZGVmYXVsdC1yb2xlcy1jb25uZWN0Il19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCJdfX0sInNjb3BlIjoib3BlbmlkIHJlYWQ6bGljZW5jZSByZWFkOmlkZW50aXR5IGVtYWlsIHByb2ZpbGUiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwiaWRDcm0iOiIxMDAwMDk4NDg2NCIsIm5hbWUiOiJSYXptaWcgU0FSS0lTU0lBTiIsInByZWZlcnJlZF91c2VybmFtZSI6InJhem1vZyIsImdpdmVuX25hbWUiOiJSYXptaWciLCJzZXNzaW9uX3N0YXRlIjoiMzk1NmMzNmUtNzMxYi00MmQ3LWI2MjYtMTYwZWJjZjZhNjZmIiwibG9jYWxlIjoiZnIiLCJmYW1pbHlfbmFtZSI6IlNBUktJU1NJQU4iLCJlbWFpbCI6InJhem1pZy5zYXJraXNzaWFuQGdtYWlsLmNvbSJ9.VSjG2htaUMt_acrqL3VcAjVMhAno9q0vdb7LTzw8UVbjIiDLzhR5msRxI8h8gSJ38kFLaa7f_SFGLIsRCSdcmhYRd2zKIrcPE-QFKbsPnH69xN2i3giMMiYEy3hj__IIyijt9z3W4KXeQdwUrlXPxprlXQ2sYTlZG63HlCGq1iI3Go9eXFmNDNM6p1jBypXcHEvJr6HwNcRdn6ZGfZ9LLMZ2aMEJAhDqL2CLrFrOZkGQpFz7ITUi_DVJAqh5DmTK1JqPswcOjhuZhDT7qWNfIleV-L7XCwvofxBwkSX9ve9l_3COZJXbsMiiRdCSTZtewlFRfgo4IuAu3g06fmJw7g; TCID=; nlbi_2712217=Ok4tKplxIEN+k1gmb9lUTgAAAAA70zbGXpiElrV2qkRjBeXO; visid_incap_2712217=LW/brcN4Rwml/7waoG/rloFBYmgAAAAAQUIPAAAAAAAlHbwlYSPbNS2qq3UBZNK8; TCPID=125629554310878226394; xtvrn=$548419$" + +def calculate_age_from_birth_date(birth_date_str): + """ + Calculate age from birth date string (format: DD/MM/YYYY) + Returns age as integer or None if parsing fails + """ + if not birth_date_str: + return None + + try: + # Parse French date format DD/MM/YYYY + birth_date = datetime.strptime(birth_date_str, '%d/%m/%Y') + today = datetime.now() + age = today.year - birth_date.year - ((today.month, today.day) < (birth_date.month, birth_date.day)) + return age + except (ValueError, TypeError): + return None + +def find_best_license_match(license_results, player_age_sportif): + """ + Find the best matching license from multiple results using ageSportif comparison + + Args: + license_results: List of license data from API + player_age_sportif: Age from ranking data + + Returns: + Tuple of (best_match, match_info) + """ + if not license_results: + return None, {"reason": "no_results"} + + if len(license_results) == 1: + return license_results[0], {"reason": "single_result", "age_match": "n/a"} + + # If we don't have ageSportif from ranking, take the first match + if player_age_sportif is None: + return license_results[0], {"reason": "no_age_data", "used_first_result": True} + + best_match = None + best_age_diff = float('inf') + match_details = [] + + for i, license_data in enumerate(license_results): + birth_date_fr = license_data.get('dateNaissanceFr') + calculated_age = calculate_age_from_birth_date(birth_date_fr) + + match_detail = { + "index": i, + "dateNaissanceFr": birth_date_fr, + "calculated_age": calculated_age, + "player_age_sportif": player_age_sportif, + "age_difference": None + } + + if calculated_age is not None: + age_diff = abs(calculated_age - player_age_sportif) + match_detail["age_difference"] = age_diff + + if age_diff < best_age_diff: + best_age_diff = age_diff + best_match = license_data + + match_details.append(match_detail) + + # If no match found with valid age, use first result + if best_match is None: + best_match = license_results[0] + match_info = { + "reason": "no_valid_ages", + "used_first_result": True, + "match_details": match_details + } + else: + match_info = { + "reason": "age_matched", + "best_age_difference": best_age_diff, + "total_candidates": len(license_results), + "match_details": match_details + } + + return best_match, match_info + +@staff_member_required +def test_player_details_apis(request): + """ + Test different API endpoints to find player license information + """ + # Sample idCrm values from your data + test_ids = [5417299111, 9721526122] + + results = { + "timestamp": datetime.now().isoformat(), + "test_results": [] + } + + for idCrm in test_ids: + player_results = { + "idCrm": idCrm, + "tests": [] + } + + # Test 1: Try player detail endpoint + try: + url = f"https://tenup.fft.fr/back/public/v1/joueurs/{idCrm}" + response = requests.get(url, timeout=10) + player_results["tests"].append({ + "method": "GET", + "url": url, + "status_code": response.status_code, + "response_preview": response.text[:500] if response.status_code == 200 else response.text + }) + except Exception as e: + player_results["tests"].append({ + "method": "GET", + "url": url, + "error": str(e) + }) + + # Test 2: Try with different endpoint structure + try: + url = f"https://tenup.fft.fr/back/public/v1/players/{idCrm}" + response = requests.get(url, timeout=10) + player_results["tests"].append({ + "method": "GET", + "url": url, + "status_code": response.status_code, + "response_preview": response.text[:500] if response.status_code == 200 else response.text + }) + except Exception as e: + player_results["tests"].append({ + "method": "GET", + "url": url, + "error": str(e) + }) + + # Test 3: Try POST with idCrm in body + try: + url = "https://tenup.fft.fr/back/public/v1/joueurs/detail" + payload = {"idCrm": idCrm} + response = requests.post(url, json=payload, headers={'Content-Type': 'application/json'}, timeout=10) + player_results["tests"].append({ + "method": "POST", + "url": url, + "payload": payload, + "status_code": response.status_code, + "response_preview": response.text[:500] if response.status_code == 200 else response.text + }) + except Exception as e: + player_results["tests"].append({ + "method": "POST", + "url": url, + "payload": payload, + "error": str(e) + }) + + # Test 4: Try the classements endpoint with more parameters + try: + url = "https://tenup.fft.fr/back/public/v1/classements/recherche" + payload = { + "pratique": "padel", + "sexe": default_sexe, + "idCrm": idCrm + } + response = requests.post(url, json=payload, headers={'Content-Type': 'application/json'}, timeout=10) + player_results["tests"].append({ + "method": "POST", + "url": url, + "payload": payload, + "status_code": response.status_code, + "response_preview": response.text[:500] if response.status_code == 200 else response.text + }) + except Exception as e: + player_results["tests"].append({ + "method": "POST", + "url": url, + "payload": payload, + "error": str(e) + }) + + results["test_results"].append(player_results) + time.sleep(0.5) # Small delay between tests + + # Create filename with timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"fft_api_tests_{timestamp}.json" + + # Return results as downloadable JSON + http_response = HttpResponse( + json.dumps(results, indent=2, ensure_ascii=False), + content_type='application/json; charset=utf-8' + ) + http_response['Content-Disposition'] = f'attachment; filename="{filename}"' + return http_response + +@staff_member_required +def explore_fft_api_endpoints(request): + """ + Try to discover FFT API endpoints + """ + base_url = "https://tenup.fft.fr/back/public/v1" + + # Common API endpoint patterns to try + endpoints_to_test = [ + "/joueurs", + "/players", + "/licences", + "/licenses", + "/classements", + "/rankings", + "/pratiques", + "/sports", + "/search", + "/recherche", + "/tournaments", + "/tournois" + ] + + results = { + "base_url": base_url, + "timestamp": datetime.now().isoformat(), + "endpoint_tests": [] + } + + for endpoint in endpoints_to_test: + try: + url = base_url + endpoint + response = requests.get(url, timeout=10) + results["endpoint_tests"].append({ + "endpoint": endpoint, + "url": url, + "status_code": response.status_code, + "content_type": response.headers.get('content-type', ''), + "response_preview": response.text[:300] if len(response.text) < 1000 else response.text[:300] + "..." + }) + except Exception as e: + results["endpoint_tests"].append({ + "endpoint": endpoint, + "url": url, + "error": str(e) + }) + + time.sleep(0.2) # Small delay + + # Create filename with timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"fft_api_discovery_{timestamp}.json" + + # Return results as downloadable JSON + http_response = HttpResponse( + json.dumps(results, indent=2, ensure_ascii=False), + content_type='application/json; charset=utf-8' + ) + http_response['Content-Disposition'] = f'attachment; filename="{filename}"' + return http_response + +@staff_member_required +def download_french_padel_rankings(request): + """ + Download French padel rankings from FFT API and save locally for later enrichment + """ + if request.method == 'POST': + start_tranche = int(request.POST.get('start_tranche', 1001)) + end_tranche = int(request.POST.get('end_tranche', 1222)) + save_locally = request.POST.get('save_locally', 'true') == 'true' + + try: + # API endpoint and parameters + url = "https://tenup.fft.fr/back/public/v1/classements/recherche" + headers = { + 'Content-Type': 'application/json' + } + + all_players = [] + total_tranches = end_tranche - start_tranche + 1 + successful_calls = 0 + failed_calls = 0 + + print(f"Starting to fetch tranches {start_tranche} to {end_tranche} ({total_tranches} total)...") + + for tranche in range(start_tranche, end_tranche + 1): + try: + payload = { + "pratique": "padel", + "sexe": default_sexe, + "tranche": tranche + } + + response = requests.post(url, json=payload, headers=headers, timeout=30) + + if response.status_code == 200: + json_data = response.json() + + if 'joueurs' in json_data and json_data['joueurs']: + # Add metadata to each player for enrichment tracking + for player in json_data['joueurs']: + player['source_tranche'] = tranche + player['license_lookup_status'] = 'not_attempted' + player['license_data'] = None + + all_players.extend(json_data['joueurs']) + successful_calls += 1 + print(f"Tranche {tranche}: Found {len(json_data['joueurs'])} players (Total: {len(all_players)})") + else: + print(f"Tranche {tranche}: No players found") + + else: + failed_calls += 1 + print(f"Tranche {tranche}: HTTP {response.status_code}") + + except requests.exceptions.RequestException as e: + failed_calls += 1 + print(f"Tranche {tranche}: Network error - {str(e)}") + + except Exception as e: + failed_calls += 1 + print(f"Tranche {tranche}: Unexpected error - {str(e)}") + + if tranche % 10 == 0: + time.sleep(0.1) + current_progress = tranche - start_tranche + 1 + print(f"Progress: {current_progress}/{total_tranches} tranches processed...") + + print(f"Completed! Total players found: {len(all_players)}") + print(f"Successful calls: {successful_calls}, Failed calls: {failed_calls}") + + if all_players: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + final_data = { + "metadata": { + "total_players": len(all_players), + "successful_tranches": successful_calls, + "failed_tranches": failed_calls, + "total_tranches_requested": total_tranches, + "tranche_range": f"{start_tranche}-{end_tranche}", + "download_timestamp": datetime.now().isoformat(), + "last_enrichment_update": None, + "enrichment_progress": { + "players_with_licenses": 0, + "players_without_licenses": len(all_players), + "last_processed_index": -1 + }, + "parameters": { + "pratique": "padel", + "sexe": default_sexe, + "tranche_start": start_tranche, + "tranche_end": end_tranche + } + }, + "joueurs": all_players + } + + # Save locally if requested + if save_locally: + rankings_dir = os.path.join(settings.BASE_DIR, 'data', 'rankings') + os.makedirs(rankings_dir, exist_ok=True) + + filename = f"french_padel_rankings_{start_tranche}-{end_tranche}_{timestamp}.json" + local_file_path = os.path.join(rankings_dir, filename) + + with open(local_file_path, 'w', encoding='utf-8') as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + print(f"Rankings saved locally to: {local_file_path}") + messages.success(request, f"Rankings saved locally to: {local_file_path}") + + # Create download response + download_filename = f"french_padel_rankings_{start_tranche}-{end_tranche}_{timestamp}.json" + http_response = HttpResponse( + json.dumps(final_data, indent=2, ensure_ascii=False), + content_type='application/json; charset=utf-8' + ) + http_response['Content-Disposition'] = f'attachment; filename="{download_filename}"' + return http_response + else: + messages.error(request, f"No players found in tranches {start_tranche}-{end_tranche}.") + + except Exception as e: + messages.error(request, f"Unexpected error: {str(e)}") + + csrf_token = get_token(request) + + html_template = Template(''' + + + + Download French Padel Rankings - Debug Tools + + + + +
+

๐Ÿ“Š Download French Padel Rankings

+ +
+ ๐Ÿ”„ New Workflow:
+ 1. Use this tool to download and save rankings locally
+ 2. Then use "Enrich Rankings with Licenses" to add license data
+ 3. Process is resumable - you can stop and restart enrichment +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+ +
+ + Saves to data/rankings/ directory for later license enrichment +
+ + +
+ +
+ โ† Back to Debug Tools +
+ + + ''') + + context = Context({ + 'csrf_token': csrf_token + }) + return HttpResponse(html_template.render(context)) + +@staff_member_required +def debug_tools_page(request): + """ + Simple debug tools page with download button + """ + html_content = ''' + + + + Debug Tools - Padel Club Admin + + + + +
+

๐Ÿ“ Debug Tools

+ +
+

French Padel Rankings Download

+

Download French padel rankings for men from the FFT API for tranches 1001 to 1222.

+ ๐Ÿ“ฅ Download Rankings (1001-1222) +
+ +
+

๐Ÿ”— Enhanced Rankings Tools

+

Advanced tools that combine multiple APIs for enriched data.

+ +

Enrich Rankings with License Data

+

Get rankings from one tranche and automatically lookup license information for each player.

+ ๐Ÿ”— Enrich Rankings with Licenses +
+ +
+

๐Ÿ” Player License Lookup Tools

+

Use the FFT Beach-Padel API to lookup detailed player information.

+ +

Search by Name

+

Search for players by nom (last name) and/or prenom (first name) to find their license information.

+ ๐Ÿ‘ค Search Player by Name + +

Lookup by License Number

+

Lookup detailed information for a single player license number.

+ ๐Ÿ” Single License Lookup + +

Bulk License Lookup

+

Lookup multiple player licenses at once (use carefully to avoid overloading the API).

+ ๐Ÿ“‹ Bulk License Lookup + +
+ ๐Ÿ’ก How to use: You'll need a valid sessionId (from Cookie header) and idHomologation (competition ID) from the FFT Beach-Padel website. Use browser dev tools to get these values. +
+
+ +
+

๐Ÿ”ฌ API Research Tools

+

Tools to help discover and explore the FFT API endpoints.

+ + ๐Ÿงช Test Player Detail APIs + ๐Ÿ” Explore FFT API Endpoints +
+ +

+ โ† Back to Admin Dashboard +
+ + + ''' + return HttpResponse(html_content) + +@staff_member_required +def get_player_license_info(request): + """ + Get player license information using sessionId and idHomologation + """ + if request.method == 'POST': + session_id = request.POST.get('sessionId', '').strip() + id_homologation = request.POST.get('idHomologation', '').strip() + license_id = request.POST.get('licenseId', '').strip() + + if not session_id or not id_homologation or not license_id: + messages.error(request, "sessionId, idHomologation, and licenseId are all required.") + return redirect(request.path) + + try: + # Construct the URL (equivalent to Swift's URL construction) + url = f"https://beach-padel.app.fft.fr/beachja/rechercheJoueur/licencies?idHomologation={id_homologation}&numeroLicence={license_id}" + + # Set up headers (equivalent to Swift's URLRequest headers) + headers = { + "Accept": "application/json, text/javascript, */*; q=0.01", + "Sec-Fetch-Site": "same-origin", + "Accept-Language": "fr-FR,fr;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Sec-Fetch-Mode": "cors", + "Host": "beach-padel.app.fft.fr", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15", + "Connection": "keep-alive", + "Referer": f"https://beach-padel.app.fft.fr/beachja/competitionFiche/inscrireEquipe?identifiantHomologation={id_homologation}", + "X-Requested-With": "XMLHttpRequest", + "Cookie": session_id + } + + print(f"Making request to: {url}") + print(f"Headers: {headers}") + + # Make the GET request (equivalent to Swift's URLSession.shared.data) + response = requests.get(url, headers=headers, timeout=30) + + print(f"Response status: {response.status_code}") + print(f"Raw JSON response: {response.text}") + + if response.status_code == 200: + try: + json_data = response.json() + + # Create result structure + result = { + "request_info": { + "url": url, + "license_id": license_id, + "id_homologation": id_homologation, + "timestamp": datetime.now().isoformat() + }, + "response_info": { + "status_code": response.status_code, + "headers": dict(response.headers), + "raw_response": response.text + }, + "parsed_data": json_data + } + + # Create filename with timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"player_license_info_{license_id}_{timestamp}.json" + + # Return as downloadable JSON + http_response = HttpResponse( + json.dumps(result, indent=2, ensure_ascii=False), + content_type='application/json; charset=utf-8' + ) + http_response['Content-Disposition'] = f'attachment; filename="{filename}"' + return http_response + + except json.JSONDecodeError as e: + messages.error(request, f"Failed to parse JSON response: {str(e)}") + + else: + messages.error(request, f"Request failed with status {response.status_code}: {response.text}") + + except requests.exceptions.RequestException as e: + messages.error(request, f"Network error: {str(e)}") + except Exception as e: + messages.error(request, f"Unexpected error: {str(e)}") + + csrf_token = get_token(request) + + # Default values + default_license_id = "5186803" + + html_template = Template(''' + + + + Player License Lookup - Debug Tools + + + + +
+

๐Ÿ” Player License Lookup

+ +
+ โœ… Default values loaded!
+ The form is pre-filled with your provided sessionId, idHomologation, and license ID. + You can modify any field if needed before submitting. +
+ +
+ + +
+ + + Pre-filled with your provided session ID +
+ +
+ + + Competition/event identifier (pre-filled) +
+ +
+ + + Player license number to lookup (pre-filled) +
+ + +
+ +
+ โ† Back to Debug Tools +
+ + + ''') + + context = Context({ + 'csrf_token': csrf_token, + 'default_session_id': default_session_id, + 'default_id_homologation': default_id_homologation, + 'default_license_id': default_license_id + }) + return HttpResponse(html_template.render(context)) + +@staff_member_required +def bulk_license_lookup(request): + """ + Lookup multiple license IDs at once + """ + if request.method == 'POST': + session_id = request.POST.get('sessionId', '').strip() + id_homologation = request.POST.get('idHomologation', '').strip() + license_ids_text = request.POST.get('licenseIds', '').strip() + + if not session_id or not id_homologation or not license_ids_text: + messages.error(request, "All fields are required.") + return redirect(request.path) + + # Parse license IDs (one per line or comma-separated) + license_ids = [] + for line in license_ids_text.replace(',', '\n').split('\n'): + license_id = line.strip() + if license_id: + license_ids.append(license_id) + + if not license_ids: + messages.error(request, "No valid license IDs provided.") + return redirect(request.path) + + results = { + "bulk_lookup_info": { + "total_licenses": len(license_ids), + "id_homologation": id_homologation, + "timestamp": datetime.now().isoformat() + }, + "results": [] + } + + # Headers setup (same as single lookup) + headers = { + "Accept": "application/json, text/javascript, */*; q=0.01", + "Sec-Fetch-Site": "same-origin", + "Accept-Language": "fr-FR,fr;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Sec-Fetch-Mode": "cors", + "Host": "beach-padel.app.fft.fr", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15", + "Connection": "keep-alive", + "Referer": f"https://beach-padel.app.fft.fr/beachja/competitionFiche/inscrireEquipe?identifiantHomologation={id_homologation}", + "X-Requested-With": "XMLHttpRequest", + "Cookie": session_id + } + + for i, license_id in enumerate(license_ids): + try: + url = f"https://beach-padel.app.fft.fr/beachja/rechercheJoueur/licencies?idHomologation={id_homologation}&numeroLicence={license_id}" + + print(f"Looking up license {i+1}/{len(license_ids)}: {license_id}") + + response = requests.get(url, headers=headers, timeout=30) + + result_item = { + "license_id": license_id, + "status_code": response.status_code, + "success": response.status_code == 200 + } + + if response.status_code == 200: + try: + json_data = response.json() + result_item["data"] = json_data + except json.JSONDecodeError: + result_item["error"] = "Failed to parse JSON" + result_item["raw_response"] = response.text[:500] + else: + result_item["error"] = f"HTTP {response.status_code}" + result_item["raw_response"] = response.text[:500] + + results["results"].append(result_item) + + # Small delay to avoid overwhelming the API + if i < len(license_ids) - 1: + time.sleep(0.5) + + except Exception as e: + results["results"].append({ + "license_id": license_id, + "success": False, + "error": str(e) + }) + + # Create filename with timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"bulk_license_lookup_{len(license_ids)}_licenses_{timestamp}.json" + + # Return as downloadable JSON + http_response = HttpResponse( + json.dumps(results, indent=2, ensure_ascii=False), + content_type='application/json; charset=utf-8' + ) + http_response['Content-Disposition'] = f'attachment; filename="{filename}"' + return http_response + + csrf_token = get_token(request) + + # Default values + default_license_ids = "5186803\n1234567\n2345678" + + html_template = Template(''' + + + + Bulk License Lookup - Debug Tools + + + + +
+

๐Ÿ” Bulk License Lookup

+ +
+ โœ… Default values loaded!
+ ID Homologation and sample license IDs are pre-filled. Add your session ID and modify license list as needed. +
+ +
+ โš ๏ธ Warning: This will make multiple API calls. Use responsibly to avoid overwhelming the FFT servers. +
+ +
+ + +
+ + + Copy the Cookie header from browser dev tools +
+ +
+ + + Competition/event identifier (pre-filled) +
+ +
+ + + Pre-filled with sample license IDs - modify as needed +
+ + +
+ +
+ โ† Back to Debug Tools +
+ + + ''') + + context = Context({ + 'csrf_token': csrf_token, + 'default_session_id': default_session_id, + 'default_id_homologation': default_id_homologation, + 'default_license_ids': default_license_ids + }) + return HttpResponse(html_template.render(context)) + +@staff_member_required +def search_player_by_name(request): + """ + Search for players by nom and prenom + """ + if request.method == 'POST': + session_id = request.POST.get('sessionId', '').strip() + id_homologation = request.POST.get('idHomologation', '').strip() + nom = request.POST.get('nom', '').strip() + prenom = request.POST.get('prenom', '').strip() + + if not session_id or not id_homologation: + messages.error(request, "sessionId and idHomologation are required.") + return redirect(request.path) + + if not nom and not prenom: + messages.error(request, "At least nom or prenom is required.") + return redirect(request.path) + + try: + # Construct the URL for name search + base_url = "https://beach-padel.app.fft.fr/beachja/rechercheJoueur/licencies" + params = { + "idHomologation": id_homologation + } + + # Add name parameters if provided + if nom: + params["nom"] = nom + if prenom: + params["prenom"] = prenom + + # Build query string + query_string = "&".join([f"{k}={v}" for k, v in params.items()]) + url = f"{base_url}?{query_string}" + + # Set up headers + headers = { + "Accept": "application/json, text/javascript, */*; q=0.01", + "Sec-Fetch-Site": "same-origin", + "Accept-Language": "fr-FR,fr;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Sec-Fetch-Mode": "cors", + "Host": "beach-padel.app.fft.fr", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15", + "Connection": "keep-alive", + "Referer": f"https://beach-padel.app.fft.fr/beachja/competitionFiche/inscrireEquipe?identifiantHomologation={id_homologation}", + "X-Requested-With": "XMLHttpRequest", + "Cookie": session_id + } + + print(f"Making request to: {url}") + print(f"Headers: {headers}") + + # Make the GET request + response = requests.get(url, headers=headers, timeout=30) + + print(f"Response status: {response.status_code}") + print(f"Raw JSON response: {response.text}") + + if response.status_code == 200: + try: + json_data = response.json() + + # Create result structure + result = { + "search_info": { + "url": url, + "nom": nom, + "prenom": prenom, + "id_homologation": id_homologation, + "timestamp": datetime.now().isoformat() + }, + "response_info": { + "status_code": response.status_code, + "headers": dict(response.headers), + "raw_response": response.text + }, + "parsed_data": json_data + } + + # Create filename with timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + search_term = f"{nom}_{prenom}".replace(" ", "_") + filename = f"player_search_{search_term}_{timestamp}.json" + + # Return as downloadable JSON + http_response = HttpResponse( + json.dumps(result, indent=2, ensure_ascii=False), + content_type='application/json; charset=utf-8' + ) + http_response['Content-Disposition'] = f'attachment; filename="{filename}"' + return http_response + + except json.JSONDecodeError as e: + messages.error(request, f"Failed to parse JSON response: {str(e)}") + + else: + messages.error(request, f"Request failed with status {response.status_code}: {response.text}") + + except requests.exceptions.RequestException as e: + messages.error(request, f"Network error: {str(e)}") + except Exception as e: + messages.error(request, f"Unexpected error: {str(e)}") + + csrf_token = get_token(request) + + # Default values + + html_template = Template(''' + + + + Search Player by Name - Debug Tools + + + + +
+

๐Ÿ” Search Player by Name

+ +
+ โœ… Default values loaded!
+ ID Homologation is pre-filled. Add your session ID and enter the player's name to search. +
+ +
+ โ„น๏ธ How it works:
+ โ€ข You can search by nom (last name), prenom (first name), or both
+ โ€ข Partial names should work (e.g., "DUPON" might find "DUPONT")
+ โ€ข Leave one field empty to search by the other
+ โ€ข Results will include license numbers and detailed player information +
+ +
+ + +
+ + + Copy the Cookie header from browser dev tools +
+ +
+ + + Competition/event identifier (pre-filled) +
+ +
+
+ + + Player's last name +
+ +
+ + + Player's first name +
+
+ + +
+ +

๐Ÿ“‹ Example Searches

+

Search by last name only: nom = "MARTIN", prenom = empty

+

Search by first name only: nom = empty, prenom = "Jean"

+

Search by both: nom = "DUPONT", prenom = "Pierre"

+

Partial search: nom = "DUPON" (might find "DUPONT", "DUPONTEL", etc.)

+ +
+ โ† Back to Debug Tools +
+ + + ''') + + context = Context({ + 'csrf_token': csrf_token, + 'default_session_id': default_session_id, + 'default_id_homologation': default_id_homologation + }) + return HttpResponse(html_template.render(context)) + +@staff_member_required +def enrich_rankings_with_licenses(request): + """ + Load a local rankings file and enrich players with license data (resumable) + Uses concurrent processing to speed up lookups for large files. + """ + if request.method == 'POST': + file_path = request.POST.get('file_path', '').strip() + session_id = request.POST.get('sessionId', default_session_id).strip() + id_homologation = request.POST.get('idHomologation', default_id_homologation).strip() + save_batch_size = int(request.POST.get('batch_size', 1000)) # How often to save progress + max_workers = int(request.POST.get('max_workers', 10)) # New parameter for controlling concurrency + + if not file_path or not session_id or not id_homologation: + messages.error(request, "File path, session ID, and ID homologation are required.") + return redirect(request.path) + + try: + # Load the rankings file + if not os.path.exists(file_path): + messages.error(request, f"File not found: {file_path}") + return redirect(request.path) + + print(f"Loading rankings from: {file_path}") + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + players = data.get('joueurs', []) + metadata = data.get('metadata', {}) + + if not players: + messages.error(request, "No players found in the file.") + return redirect(request.path) + + print(f"Loaded {len(players)} players from file") + + # Define a worker function that will process a single player + def lookup_player_license(player_tuple, session_id, id_homologation): + original_index, player = player_tuple + + # Extract and normalize names + raw_nom = player.get('nom') + raw_prenom = player.get('prenom') + + if not raw_nom or not raw_prenom: + player['license_lookup_status'] = 'missing_name_data' + return player, False, None + + # Keep original case and accents, just clean up any extra whitespace + nom = raw_nom.strip() + prenom = raw_prenom.strip() + + # Get player's gender + sexe = player.get('sexe', 'H') # Default to 'H' for male + + # Setup license lookup headers + license_headers = { + "Accept": "application/json, text/javascript, */*; q=0.01", + "Sec-Fetch-Site": "same-origin", + "Accept-Language": "fr-FR,fr;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Sec-Fetch-Mode": "cors", + "Host": "beach-padel.app.fft.fr", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15", + "Connection": "keep-alive", + "Referer": f"https://beach-padel.app.fft.fr/beachja/competitionFiche/inscrireEquipe?identifiantHomologation={id_homologation}", + "X-Requested-With": "XMLHttpRequest", + "Cookie": session_id + } + + def sanitize_for_latin1(text): + # Replace specific problematic characters + text = text.replace('\u2019', "'") # Replace right single quotation mark with regular apostrophe + # Add more replacements as needed + return text + + try: + # Build license lookup URL with proper URL encoding + license_url = f"https://beach-padel.app.fft.fr/beachja/rechercheJoueur/licencies?idHomologation={id_homologation}&nom={urllib.parse.quote(sanitize_for_latin1(nom), encoding='latin-1')}&prenom={urllib.parse.quote(sanitize_for_latin1(prenom), encoding='latin-1')}&sexe={sexe}" + + # Make license lookup request + license_response = requests.get(license_url, headers=license_headers, timeout=30) + license_response.encoding = 'utf-8' # Ensure consistent encoding + + if license_response.status_code == 200: + try: + license_data = license_response.json() + + if license_data and 'object' in license_data and 'listeJoueurs' in license_data['object']: + liste_joueurs = license_data['object']['listeJoueurs'] + presence_doublon = license_data['object'].get('presenceDoublon', False) + + if liste_joueurs: + # Get player's age from ranking data for duplicate matching + player_age_sportif = player.get('ageSportif') + + # Find the best matching license using age comparison + license_info, match_info = find_best_license_match(liste_joueurs, player_age_sportif) + + # Add all license data to player + player['licence'] = license_info.get('licence') + player['codeClub'] = license_info.get('codeClub') + player['nomClub'] = license_info.get('nomClub') + + # Track duplicates and matching info + player['presenceDoublon'] = presence_doublon + player['nombreResultatsLicence'] = len(liste_joueurs) + player['license_match_info'] = match_info + + if len(liste_joueurs) > 1: + player['tousResultatsLicence'] = liste_joueurs + + player['license_lookup_status'] = 'success' + + # Enhanced logging with age matching info + doublon_status = "" + if presence_doublon: + if match_info["reason"] == "age_matched": + age_diff = match_info.get("best_age_difference", "?") + doublon_status = f" (DOUBLON - matched by age, diff: {age_diff})" + elif match_info["reason"] == "no_age_data": + doublon_status = " (DOUBLON - no age data, used first)" + else: + doublon_status = f" (DOUBLON - {match_info['reason']})" + + return player, True, f"โœ… {nom} {prenom} -> {license_info.get('licence')}{doublon_status}" + else: + player['license_lookup_status'] = 'no_results' + player['presenceDoublon'] = presence_doublon + return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> No results" + else: + player['license_lookup_status'] = 'no_data' + return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> No data" + except json.JSONDecodeError as json_err: + player['license_lookup_status'] = 'json_decode_error' + player['license_lookup_error'] = str(json_err) + player['raw_response'] = license_response.text + + # Debug info for JSON decode errors + error_position = json_err.pos + return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> JSON decode error: {str(json_err)} at pos {error_position}" + else: + player['license_lookup_status'] = f'http_error_{license_response.status_code}' + return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> HTTP {license_response.status_code}" + + except requests.exceptions.RequestException as e: + player['license_lookup_status'] = f'network_error: {type(e).__name__}' + player['license_lookup_error'] = str(e) + return player, False, f"Failed {nom} {prenom} {player['idCrm']} -> Network error: {type(e).__name__}: {str(e)}" + + # Find players that need license enrichment + players_needing_licenses = [] + players_with_licenses = 0 + + for i, player in enumerate(players): + if player is None: + continue + if player.get('license_lookup_status') not in ['success']: + players_needing_licenses.append((i, player)) + else: + players_with_licenses += 1 + + print(f"Players with licenses: {players_with_licenses}") + print(f"Players needing licenses: {len(players_needing_licenses)}") + + if not players_needing_licenses: + messages.success(request, "All players already have license data!") + return redirect(request.path) + + # Process players concurrently in batches + total_processed = 0 + successful_lookups = 0 + failed_lookups = 0 + + print(f"Starting concurrent enrichment of {len(players_needing_licenses)} players with {max_workers} workers") + + # Prepare the partial function with fixed parameters + worker_fn = partial(lookup_player_license, session_id=session_id, id_homologation=id_homologation) + + # Process in batches for saving progress, using the value from the form + for batch_start in range(0, len(players_needing_licenses), save_batch_size): + batch_end = min(batch_start + save_batch_size, len(players_needing_licenses)) + current_batch = players_needing_licenses[batch_start:batch_end] + + # print(f"\n=== Processing batch {batch_start//save_batch_size + 1}: players {batch_start+1} to {batch_end} ===") + batch_results = [] + + # Process the current batch with concurrent workers + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_player = {executor.submit(worker_fn, player_tuple): player_tuple for player_tuple in current_batch} + + for future in concurrent.futures.as_completed(future_to_player): + player_tuple = future_to_player[future] + original_index = player_tuple[0] + + try: + updated_player, success, message = future.result() + players[original_index] = updated_player # Update the player in the main list + + if success: + successful_lookups += 1 + else: + failed_lookups += 1 + + total_processed += 1 + + # Print progress every 10 players + # if total_processed % 10 == 0 or total_processed == len(players_needing_licenses): + # print(f" Progress: {total_processed}/{len(players_needing_licenses)} ({(total_processed/len(players_needing_licenses)*100):.1f}%)") + + if message: + batch_results.append(message) + + except Exception as e: + print(f" ERROR processing player {original_index} ({player_tuple}'): {str(e)}") + failed_lookups += 1 + total_processed += 1 + + # Print batch results + for msg in batch_results: + print(f" {msg}") + + # Save progress after each batch + metadata['last_enrichment_update'] = datetime.now().isoformat() + metadata['enrichment_progress'] = { + 'players_with_licenses': successful_lookups + players_with_licenses, + 'players_without_licenses': len(players) - (successful_lookups + players_with_licenses), + 'last_processed_index': batch_end - 1, + 'total_processed_this_run': total_processed, + 'successful_this_run': successful_lookups, + 'failed_this_run': failed_lookups + } + + # Save the updated file + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + # print(f"Progress saved: {successful_lookups + players_with_licenses}/{len(players)} players have licenses") + + print(f"\n=== ENRICHMENT COMPLETE ===") + print(f"Total processed this run: {total_processed}") + print(f"Successful lookups this run: {successful_lookups}") + print(f"Failed lookups this run: {failed_lookups}") + print(f"Total players with licenses: {successful_lookups + players_with_licenses}/{len(players)}") + + messages.success(request, f"Enrichment complete! Processed {total_processed} players. {successful_lookups + players_with_licenses}/{len(players)} players now have license data.") + + except Exception as e: + import traceback + error_traceback = traceback.format_exc() + print(f"CRITICAL ERROR: {type(e).__name__}: {str(e)}") + print(f"Traceback: {error_traceback}") + messages.error(request, f"Error during enrichment: {type(e).__name__}: {str(e)}") + return redirect(request.path) + + # Show the form + from django.template import Template, Context + from django.middleware.csrf import get_token + + csrf_token = get_token(request) + + # Try to find existing ranking files + rankings_dir = os.path.join(settings.BASE_DIR, 'data', 'rankings') + existing_files = [] + if os.path.exists(rankings_dir): + for filename in os.listdir(rankings_dir): + if filename.endswith('.json') and 'french_padel_rankings' in filename: + file_path = os.path.join(rankings_dir, filename) + existing_files.append({ + 'filename': filename, + 'path': file_path, + 'size': os.path.getsize(file_path), + 'modified': datetime.fromtimestamp(os.path.getmtime(file_path)).strftime('%Y-%m-%d %H:%M:%S') + }) + + existing_files.sort(key=lambda x: x['modified'], reverse=True) + + html_template = Template(''' + + + + Enrich Rankings with Licenses - Debug Tools + + + + +
+

Enrich Rankings with Licenses

+ +
+

Optimized for large datasets: This tool now supports concurrent processing to speed up license lookups.

+

You can adjust the batch size and number of concurrent workers based on your server's capacity.

+
+ +
+ {% csrf_token %} + +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + + How often to save progress to disk. After processing this many players, the file will be updated with results. +
+ +
+ + + Higher values = faster processing but more server load. Adjust based on server capacity. +
+ + +
+ +

Available Ranking Files

+
+ {% for file in existing_files %} +
+ {{ file.filename }}
+ Path: {{ file.path }}
+ Size: {{ file.size|filesizeformat }}
+ Modified: {{ file.modified }} + +
+ {% empty %} +

No ranking files found in the data/rankings directory.

+ {% endfor %} +
+ + โ† Back to Debug Tools +
+ + + + + ''') + + context = { + 'csrf_token': csrf_token, + 'existing_files': existing_files, + 'default_session_id': default_session_id, + 'default_id_homologation': default_id_homologation, + } + + rendered_html = html_template.render(Context(context)) + return HttpResponse(rendered_html) diff --git a/tournaments/management/commands/analyze_rankings.py b/tournaments/management/commands/analyze_rankings.py new file mode 100644 index 0000000..27935fb --- /dev/null +++ b/tournaments/management/commands/analyze_rankings.py @@ -0,0 +1,1057 @@ +from django.core.management.base import BaseCommand, CommandError +import os +import csv +import collections +import re +from datetime import datetime +from django.conf import settings +import json +import tempfile +import shutil + +class Command(BaseCommand): + help = 'Analyze a padel rankings CSV file and provide statistics' + + def add_arguments(self, parser): + parser.add_argument('file_path', nargs='?', type=str, help='Relative path to the rankings file from the static/rankings directory') + parser.add_argument('--full-path', type=str, help='Full path to the rankings file (alternative to file_path)') + parser.add_argument('--list-files', action='store_true', help='List available ranking files') + parser.add_argument('--top', type=int, default=10, help='Number of top players to display') + parser.add_argument('--clubs', type=int, default=10, help='Number of top clubs to display') + parser.add_argument('--leagues', type=int, default=10, help='Number of top leagues to display') + parser.add_argument('--find-anonymous', action='store_true', help='Find and match anonymous players with previous month rankings') + parser.add_argument('--confidence', type=float, default=0.7, help='Confidence threshold for automatic matching (0-1)') + parser.add_argument('--auto-match', action='store_true', help='Automatically match anonymous players when confidence is high') + parser.add_argument('--output', type=str, help='Save results to output file') + parser.add_argument('--verbose', action='store_true', help='Show detailed matching information') + + def handle(self, *args, **options): + # Base directory for rankings files + rankings_dir = os.path.join(settings.BASE_DIR, 'tournaments', 'static', 'rankings') + + # Check if user wants to list available files + if options['list_files']: + self.list_available_files(rankings_dir) + return + + # Get the file path + if options['full_path']: + file_path = options['full_path'] + elif options['file_path']: + file_path = os.path.join(rankings_dir, options['file_path']) + else: + self.stderr.write(self.style.ERROR('Please provide a file path or use --list-files to see available files')) + return + + # Validate file exists + if not os.path.exists(file_path): + self.stderr.write(self.style.ERROR(f'File not found: {file_path}')) + return + + # Process the file + players, metadata = self.parse_rankings_file(file_path) + + # Generate statistics + if players: + self.generate_statistics(players, options) + + # Find anonymous players if requested + if options['find_anonymous']: + if options['auto_match']: + # Iterative approach: keep matching until no more changes can be made + self.iterative_match_anonymous_players(file_path, rankings_dir, options) + else: + # Single pass analysis without making changes + self.find_anonymous_players(players, metadata, rankings_dir, options, file_path) + + def list_available_files(self, rankings_dir): + """List all available ranking files""" + if not os.path.exists(rankings_dir): + self.stderr.write(self.style.ERROR(f'Rankings directory not found: {rankings_dir}')) + return + + files = [f for f in os.listdir(rankings_dir) if f.endswith('.csv')] + files.sort() + + self.stdout.write(self.style.SUCCESS(f'Found {len(files)} ranking files:')) + for f in files: + self.stdout.write(f' - {f}') + + def parse_rankings_file(self, file_path): + """Parse a rankings file and return player data and metadata""" + try: + self.stdout.write(f"Loading file: {file_path}...") + + # Read the file and parse data + with open(file_path, 'r', encoding='utf-8') as f: + lines = f.readlines() + + self.stdout.write(f"File loaded. Found {len(lines)} lines, processing...") + + # Extract file metadata from first lines + title = lines[0].strip().strip('"') + period = lines[1].strip().strip('"') + + # Parse month and year from filename or content + filename = os.path.basename(file_path) + + # Extract month-year from filename (format: CLASSEMENT-PADEL-MESSIEURS-MM-YYYY.csv) + match = re.search(r'(\d{2})-(\d{4})', filename) + if match: + month = int(match.group(1)) + year = int(match.group(2)) + else: + # Try to extract from period + match = re.search(r'(\w+)\s+(\d{4})', period) + if match: + month_name = match.group(1) + month_names = ["JANVIER", "FEVRIER", "MARS", "AVRIL", "MAI", "JUIN", + "JUILLET", "AOUT", "SEPTEMBRE", "OCTOBRE", "NOVEMBRE", "DECEMBRE"] + if month_name.upper() in month_names: + month = month_names.index(month_name.upper()) + 1 + else: + month = datetime.now().month + year = int(match.group(2)) + else: + # Default to current + month = datetime.now().month + year = datetime.now().year + + # Extract gender from filename + gender = "UNKNOWN" + if "MESSIEURS" in filename: + gender = "MESSIEURS" + elif "DAMES" in filename: + gender = "DAMES" + + # Extract tranche/series from filename (e.g., MESSIEURS-2 or MESSIEURS-3) + tranche = None + tranche_match = re.search(r'MESSIEURS-(\d)', filename) + if tranche_match: + tranche = int(tranche_match.group(1)) + + metadata = { + 'title': title, + 'period': period, + 'filename': filename, + 'month': month, + 'year': year, + 'gender': gender, + 'tranche': tranche + } + + self.stdout.write(self.style.SUCCESS(f'Analyzing: {title} - {period}')) + + # Find the actual data start (after header rows) + data_start = 0 + for i, line in enumerate(lines): + if ';RANG;NOM;PRENOM;' in line: + data_start = i + 1 + header = line.strip().split(';') + break + + # Parse player data + self.stdout.write(f"Parsing player data from line {data_start}...") + players = [] + line_count = 0 + total_lines = len(lines[data_start:]) + progress_interval = max(1, total_lines // 10) # Report progress at 10% intervals + + for line in lines[data_start:]: + if not line.strip(): + continue + + values = line.strip().split(';') + if len(values) < 5: # Skip malformed lines + continue + + # Create player record based on the Swift code line format + # ";\(rank);\(lastName);\(firstName);\(country);\(strippedLicense);\(pointsString);\(assimilation); + # \(tournamentCountString);\(ligue);\(formatNumbers(clubCode));\(club);\(progression.formattedAsRawString()); + # \(bestRank?.formattedAsRawString() ?? "");\(birthYear?.formattedAsRawString() ?? "");" + + player = { + 'rank': values[1].strip() if len(values) > 1 and values[1].strip() else 'N/A', + 'name': values[2].strip() if len(values) > 2 and values[2].strip() else 'N/A', + 'first_name': values[3].strip() if len(values) > 3 and values[3].strip() else 'N/A', + 'nationality': values[4].strip() if len(values) > 4 and values[4].strip() else 'N/A', + 'license': values[5].strip() if len(values) > 5 and values[5].strip() else 'N/A', + 'points': values[6].strip() if len(values) > 6 and values[6].strip() else 'N/A', + 'assimilation': values[7].strip() if len(values) > 7 and values[7].strip() else 'N/A', + 'tournaments_played': values[8].strip() if len(values) > 8 and values[8].strip() else 'N/A', + 'league': values[9].strip() if len(values) > 9 and values[9].strip() else 'N/A', + 'club_code': values[10].strip() if len(values) > 10 and values[10].strip() else 'N/A', + 'club': values[11].strip() if len(values) > 11 and values[11].strip() else 'N/A', + 'progression': values[12].strip() if len(values) > 12 and values[12].strip() else '0', + 'best_rank': values[13].strip() if len(values) > 13 and values[13].strip() else 'N/A', + 'birth_year': values[14].strip() if len(values) > 14 and values[14].strip() else 'N/A', + } + players.append(player) + + # Show progress periodically + line_count += 1 + if line_count % progress_interval == 0: + self.stdout.write(f" Progress: {line_count}/{total_lines} lines processed ({(line_count/total_lines)*100:.1f}%)") + + return players, metadata + + except Exception as e: + self.stderr.write(self.style.ERROR(f'Error parsing file: {str(e)}')) + return [], {} + + def generate_statistics(self, players, options): + """Generate and display statistics about the ranking data""" + total_players = len(players) + self.stdout.write(f'Total players: {total_players}') + + # Top players + self.stdout.write(self.style.SUCCESS(f'\nTop {options["top"]} players:')) + for i, player in enumerate(players[:options["top"]]): + progression = f" ({player['progression']})" if player['progression'] != 'N/A' else "" + self.stdout.write(f'{i+1}. {player["rank"]} - {player["name"]} {player["first_name"]} - {player["points"]} points{progression}') + + # League distribution + league_counter = collections.Counter([p["league"] for p in players if p["league"] != 'N/A']) + self.stdout.write(self.style.SUCCESS(f'\nPlayers by league (top {options["leagues"]}):')) + for league, count in league_counter.most_common(options["leagues"]): + percentage = (count / total_players) * 100 + self.stdout.write(f'{league}: {count} players ({percentage:.1f}%)') + + # Club distribution + club_counter = collections.Counter([p["club"] for p in players if p["club"] != 'N/A']) + self.stdout.write(self.style.SUCCESS(f'\nPlayers by club (top {options["clubs"]}):')) + for club, count in club_counter.most_common(options["clubs"]): + percentage = (count / total_players) * 100 + self.stdout.write(f'{club}: {count} players ({percentage:.1f}%)') + + # Points statistics (if numeric) + try: + points = [float(p["points"]) for p in players if p["points"] not in ('N/A', '')] + if points: + self.stdout.write(self.style.SUCCESS('\nPoints statistics:')) + self.stdout.write(f'Min: {min(points)}') + self.stdout.write(f'Max: {max(points)}') + self.stdout.write(f'Average: {sum(points) / len(points):.2f}') + self.stdout.write(f'Players with points: {len(points)} ({(len(points) / total_players) * 100:.1f}%)') + except ValueError: + # Points might not be numeric + pass + + # Anonymous players count + anonymous_players = [p for p in players if self.is_anonymous_player(p)] + if anonymous_players: + self.stdout.write(self.style.SUCCESS(f'\nAnonymous players: {len(anonymous_players)} ({(len(anonymous_players) / total_players) * 100:.1f}%)')) + + def is_anonymous_player(self, player): + """Check if a player is anonymous (missing name data)""" + # Define criteria for anonymous players - adjust as needed + return (player['name'] == 'N/A' or player['name'] == '' or + player['first_name'] == 'N/A' or player['first_name'] == '') + + def player_exists_in_current_month(self, prev_player, current_players_indexes): + """ + Check if a player from the previous month already exists in the current month. + Uses pre-built indexes for fast lookup. + + Args: + prev_player: Player from previous month + current_players_indexes: Dictionary of indexes for fast lookup + + Returns: + (exists, matching_player) tuple + """ + # 1. Check by license number (fastest) + if prev_player['license'] != 'N/A' and prev_player['license']: + license_index = current_players_indexes.get('license_index', {}) + if prev_player['license'] in license_index: + return True, license_index[prev_player['license']] + + return False, None + + def build_current_players_indexes(self, current_players): + """ + Pre-process current players into lookup indexes for faster duplicate checking. + Returns a dictionary of indexes. + """ + self.stdout.write("Building player indexes for fast lookup...") + start_time = datetime.now() + + # Players to index (only non-anonymous) + players_to_index = [p for p in current_players if not self.is_anonymous_player(p)] + + # Create license index + license_index = {} + for player in players_to_index: + if player['license'] != 'N/A' and player['license']: + license_index[player['license']] = player + + # Create name index + name_index = {} + for player in players_to_index: + if player['name'] != 'N/A' and player['first_name'] != 'N/A': + name_key = f"{player['name'].lower()}_{player['first_name'].lower()}" + name_index[name_key] = player + + # Create name+club/league index + name_club_league_index = {} + for player in players_to_index: + if player['name'] != 'N/A': + # Name + club + if player['club'] != 'N/A': + name_club_key = f"{player['name'].lower()}_{player['club'].lower()}" + name_club_league_index[name_club_key] = player + + # Name + league + if player['league'] != 'N/A': + name_league_key = f"{player['name'].lower()}_{player['league'].lower()}" + name_club_league_index[name_league_key] = player + + indexes = { + 'license_index': license_index, + 'name_index': name_index, + 'name_club_league_index': name_club_league_index + } + + elapsed = (datetime.now() - start_time).total_seconds() + self.stdout.write(f"Indexes built in {elapsed:.2f} seconds. License keys: {len(license_index)}, Name keys: {len(name_index)}") + + return indexes + + def find_previous_month_file(self, current_metadata, rankings_dir): + """Find the rankings file for the previous month""" + current_month = current_metadata['month'] + current_year = current_metadata['year'] + gender = current_metadata['gender'] + tranche = current_metadata['tranche'] + + # Calculate previous month and year + prev_month = current_month - 1 + prev_year = current_year + if prev_month == 0: + prev_month = 12 + prev_year = current_year - 1 + + # Format for filename pattern + tranche_part = f"-{tranche}" if tranche else "" + pattern = f"CLASSEMENT-PADEL-{gender}{tranche_part}-{prev_month:02d}-{prev_year}.csv" + + # Look for exact match first + exact_path = os.path.join(rankings_dir, pattern) + if os.path.exists(exact_path): + return exact_path + + # Otherwise, try more fuzzy matching + pattern_base = f"CLASSEMENT-PADEL-{gender}{tranche_part}-{prev_month:02d}" + for filename in os.listdir(rankings_dir): + if filename.startswith(pattern_base) and filename.endswith(".csv"): + return os.path.join(rankings_dir, filename) + + # If still not found, look for any file from previous month + pattern_fallback = f"CLASSEMENT-PADEL-{gender}-{prev_month:02d}" + for filename in os.listdir(rankings_dir): + if filename.startswith(pattern_fallback) and filename.endswith(".csv"): + return os.path.join(rankings_dir, filename) + + return None + + + def find_anonymous_players(self, current_players, current_metadata, rankings_dir, options, file_path=None, return_count=False): + """ + Find anonymous players and try to match them with players from previous month. + + Args: + current_players: List of current month players + current_metadata: Metadata about current month file + rankings_dir: Directory containing ranking files + options: Command options + file_path: Path to current month file (for auto-match) + return_count: Whether to return the count of matched players + + Returns: + Number of matched players if return_count is True, otherwise None + """ + start_time = datetime.now() + + # Initialize matched_count + matched_count = 0 + + # Identify anonymous players + anonymous_players = [p for p in current_players if self.is_anonymous_player(p)] + if not anonymous_players: + self.stdout.write(self.style.SUCCESS('No anonymous players found!')) + if return_count: + return 0 + return + + self.stdout.write(self.style.SUCCESS(f'\nFound {len(anonymous_players)} anonymous players. Looking for matches...')) + + # Find previous month file + prev_month_file = self.find_previous_month_file(current_metadata, rankings_dir) + if not prev_month_file: + self.stderr.write(self.style.ERROR('Previous month rankings file not found!')) + if return_count: + return 0 + return + + self.stdout.write(f'Using previous month file: {os.path.basename(prev_month_file)}') + + # Load previous month data + self.stdout.write('Loading previous month data...') + prev_players, prev_metadata = self.parse_rankings_file(prev_month_file) + if not prev_players: + self.stderr.write(self.style.ERROR('Could not load previous month data!')) + if return_count: + return 0 + return + + # Build fast lookup indexes for current players (major performance optimization) + current_players_indexes = self.build_current_players_indexes(current_players) + + # Track potential matches + matches_found = 0 + high_confidence_matches = 0 + skipped_existing_players = 0 + results = [] + + # For each anonymous player, try to find matches + self.stdout.write(f'Analyzing {len(anonymous_players)} anonymous players...') + progress_counter = 0 + progress_interval = max(1, len(anonymous_players) // 10) # Report progress at 10% intervals + + for anon_player in anonymous_players: + # Show progress + progress_counter += 1 + if progress_counter % progress_interval == 0 or progress_counter == 1: + self.stdout.write(f' Processing anonymous player {progress_counter}/{len(anonymous_players)} ({(progress_counter/len(anonymous_players))*100:.1f}%)') + + potential_matches = self.find_potential_matches(anon_player, prev_players, current_players_indexes, options) + + if potential_matches: + matches_found += 1 + best_match = potential_matches[0] # Highest confidence match + + # Record the match info + match_info = { + 'anonymous_player': anon_player, + 'potential_matches': potential_matches, + 'best_match': best_match + } + results.append(match_info) + + # Output match information + progression = f", Progression: {anon_player['progression']}" if anon_player['progression'] != 'N/A' else "" + assimilation = f", Assimilation: {anon_player['assimilation']}" if anon_player['assimilation'] != 'N/A' else "" + + self.stdout.write(f"\nAnonymous player: Rank {anon_player['rank']}, League: {anon_player['league']}{progression}{assimilation}") + + for i, match in enumerate(potential_matches[:3]): # Show top 3 matches + player = match['player'] + confidence = match['confidence'] + match_reasons = match['match_reasons'] + self.stdout.write(f" Match {i+1}: {player['name']} {player['first_name']} (Rank: {player['rank']}, League: {player['league']})") + self.stdout.write(f" Confidence: {confidence:.2f}, Match reasons: {match_reasons}") + + # Count high confidence matches + if best_match['confidence'] >= options['confidence']: + high_confidence_matches += 1 + else: + if options['verbose']: + self.stdout.write(f"\nNo matches found for anonymous player: Rank {anon_player['rank']}, League: {anon_player['league']}") + + # Batch processing status update + if progress_counter % 100 == 0 and progress_counter > 0: + elapsed = (datetime.now() - start_time).total_seconds() + per_player = elapsed / progress_counter + remaining = (len(anonymous_players) - progress_counter) * per_player + self.stdout.write(f" Processed {progress_counter}/{len(anonymous_players)} players in {elapsed:.1f}s") + self.stdout.write(f" Estimated time remaining: {remaining:.1f}s ({per_player:.3f}s per player)") + + # Final timing + total_elapsed = (datetime.now() - start_time).total_seconds() + self.stdout.write(f"Analysis completed in {total_elapsed:.2f} seconds ({total_elapsed/len(anonymous_players):.3f}s per player)") + + # Summary + self.stdout.write(self.style.SUCCESS(f'\nMatching summary:')) + self.stdout.write(f'Total anonymous players: {len(anonymous_players)}') + self.stdout.write(f'Players with potential matches: {matches_found}') + self.stdout.write(f'High confidence matches (โ‰ฅ{options["confidence"]}): {high_confidence_matches}') + self.stdout.write(f'Skipped players already in current month: {skipped_existing_players}') + + # Save results if requested + if options['output']: + self.stdout.write(f'Saving results to {options["output"]}...') + self.save_results(results, options['output']) + + # Auto-match players if requested + if options['auto_match'] and matches_found > 0 and file_path: + matched_count = self.update_rankings_with_matches(file_path, anonymous_players, results, + options['confidence'], options) + elif options['auto_match'] and file_path is None: + self.stderr.write(self.style.ERROR("Auto-match was requested but file path is not available. No changes were made.")) + + # Return matched count if requested + if return_count: + return matched_count + return None + + def find_potential_matches(self, anon_player, prev_players, current_players_indexes, options): + """Find potential matches for an anonymous player from previous month data""" + start_time = datetime.now() + potential_matches = [] + skipped_players = 0 + + # Show what we're matching + if options['verbose']: + progression = f", Progression: {anon_player['progression']}" if anon_player['progression'] != 'N/A' else "" + self.stdout.write(f" Finding matches for anonymous player: Rank {anon_player['rank']}{progression}, League: {anon_player['league']}") + + # Get ranking as integer if possible + try: + anon_rank = int(anon_player['rank']) if anon_player['rank'] != 'N/A' else None + except ValueError: + anon_rank = None + + # Parse progression to get previous rank if available + prev_rank_from_progression = None + prog_value = 0 # Default if no progression + + if anon_player['progression'] != 'N/A' and anon_player['progression']: + try: + # Progression can be like "+5", "-10", "=", etc. + prog_str = anon_player['progression'].strip() + if prog_str.startswith('+'): + # CRITICAL FIX: If progression is positive (e.g., +96), player moved UP by 96 places + # So previous rank is HIGHER (current rank + progression) + prog_value = int(prog_str) + + elif prog_str.startswith('-'): + # If progression is negative (e.g., -10), player moved DOWN by 10 places + # So previous rank is LOWER (current rank + progression) + prog_value = int(prog_str) + + elif prog_str == '=': + prog_value = 0 + + # Handle pure numeric progression without sign + elif prog_str.isdigit() or (prog_str.isdigit() and prog_str.startswith('-')): + prog_value = int(prog_str) + + # Default to 0 for "NEW" or other special values + except ValueError: + prog_value = 0 + + # Calculate expected previous rank + if anon_rank is not None: + prev_rank_from_progression = anon_rank + prog_value # Add progression for previous rank + if options['verbose']: + self.stdout.write(f" Target previous rank: {prev_rank_from_progression} (current rank {anon_rank} + progression {prog_value})") + + for prev_player in prev_players: + # Skip anonymous players in previous month + if self.is_anonymous_player(prev_player): + continue + + # Check if this player exists in current month with the same license + exists, existing_player = self.player_exists_in_current_month(prev_player, current_players_indexes) + if exists: + # If we found the exact same player (same license), skip them + if existing_player['license'] == prev_player['license']: + skipped_players += 1 + continue + # If we found someone with the same name but different license, we can still consider this player + + # Initialize match data + match_data = { + 'player': prev_player, + 'rank_match_type': None, + 'rank_diff': None, + 'has_league_match': False, + 'has_assimilation_match': False, + 'points_similarity': 0.0, + 'match_reasons': [], + 'confidence': 0.0 + } + + # 1. PRIMARY MATCHER: Previous rank match + if prev_rank_from_progression is not None: + try: + prev_rank_value = int(prev_player['rank']) + if prev_rank_value is not None: + rank_diff = abs(prev_rank_value - prev_rank_from_progression) + match_data['rank_diff'] = rank_diff + + if rank_diff == 0: + match_data['rank_match_type'] = 'exact' + match_data['match_reasons'].append(f"exact previous rank match ({prev_rank_value})") + match_data['confidence'] = 0.7 + elif rank_diff <= 3: + match_data['rank_match_type'] = 'close' + match_data['match_reasons'].append(f"close previous rank match ({prev_rank_value} vs {prev_rank_from_progression})") + match_data['confidence'] = 0.4 + elif rank_diff <= 10: + match_data['rank_match_type'] = 'approximate' + match_data['match_reasons'].append(f"approximate previous rank match ({prev_rank_value} vs {prev_rank_from_progression})") + match_data['confidence'] = 0.2 + except ValueError: + pass + + # 2. Points similarity (new check) + try: + if anon_player['points'] != 'N/A' and prev_player['points'] != 'N/A': + anon_points = float(anon_player['points']) + prev_points = float(prev_player['points']) + points_diff = abs(anon_points - prev_points) + points_similarity = max(0, 1 - (points_diff / max(anon_points, prev_points))) + + if points_similarity > 0.9: + match_data['points_similarity'] = points_similarity + match_data['match_reasons'].append(f"similar points ({prev_points} vs {anon_points})") + match_data['confidence'] += 0.2 + except ValueError: + pass + + # 3. League match + if anon_player['league'] != 'N/A' and prev_player['league'] != 'N/A': + if anon_player['league'] == prev_player['league']: + match_data['has_league_match'] = True + match_data['match_reasons'].append("league match") + match_data['confidence'] += 0.25 + + # 4. Assimilation match + if anon_player['assimilation'] != 'N/A' and prev_player['assimilation'] != 'N/A': + if anon_player['assimilation'] == prev_player['assimilation']: + match_data['has_assimilation_match'] = True + match_data['match_reasons'].append("assimilation match") + match_data['confidence'] += 0.1 + + # Only consider matches with minimum confidence + if match_data['confidence'] >= 0.1: + match_data['match_reasons'] = ", ".join(match_data['match_reasons']) + potential_matches.append(match_data) + + # Sort matches with updated criteria + def match_sort_key(match): + rank_score = { + 'exact': 1000, + 'close': 100, + 'approximate': 10, + None: 1 + }.get(match['rank_match_type'], 0) + + points_score = int(match.get('points_similarity', 0) * 100) + league_value = 2 if match['has_league_match'] else 1 + assimilation_value = 2 if match['has_assimilation_match'] else 1 + + return (rank_score, points_score, league_value, assimilation_value, match['confidence']) + + potential_matches.sort(key=match_sort_key, reverse=True) + return potential_matches + + def save_results(self, results, output_path): + """Save matching results to a file""" + try: + with open(output_path, 'w', encoding='utf-8') as f: + f.write("Anonymous Player Matching Results\n") + f.write("================================\n\n") + + for match_info in results: + anon_player = match_info['anonymous_player'] + best_match = match_info['best_match'] + + progression = f", Progression: {anon_player['progression']}" if anon_player['progression'] != 'N/A' else "" + assimilation = f", Assimilation: {anon_player['assimilation']}" if anon_player['assimilation'] != 'N/A' else "" + + f.write(f"Anonymous Player (Rank: {anon_player['rank']}, League: {anon_player['league']}{progression}{assimilation})\n") + f.write(f"Best Match: {best_match['player']['name']} {best_match['player']['first_name']}\n") + f.write(f" Confidence: {best_match['confidence']:.2f}\n") + f.write(f" Match reasons: {best_match['match_reasons']}\n") + f.write(f" Previous Rank: {best_match['player']['rank']}\n") + f.write(f" League: {best_match['player']['league']}\n") + f.write(f" Club: {best_match['player']['club']}\n\n") + + self.stdout.write(self.style.SUCCESS(f'Results saved to {output_path}')) + except Exception as e: + self.stderr.write(self.style.ERROR(f'Error saving results: {str(e)}')) + + def update_rankings_with_matches(self, file_path, anonymous_players, matches, confidence_threshold, options): + """ + Update the rankings file with matched player information + + Args: + file_path: Path to the current month's rankings file + anonymous_players: List of anonymous players + matches: List of match info dictionaries + confidence_threshold: Minimum confidence to apply auto-matching + options: Command options + + Returns: + Number of players that were updated + """ + self.stdout.write(self.style.SUCCESS(f"\nAuto-matching players with confidence โ‰ฅ {confidence_threshold}...")) + + # Create a backup of the original file + backup_path = f"{file_path}.bak" + shutil.copy2(file_path, backup_path) + self.stdout.write(f"Created backup of original file at: {backup_path}") + + # Read the original file + with open(file_path, 'r', encoding='utf-8') as f: + lines = f.readlines() + + # Create a map of anonymous players by rank for faster lookup + anon_by_rank = {} + for player in anonymous_players: + if player['rank'] != 'N/A': + anon_by_rank[player['rank']] = player + + # Track which players will be updated (use a dictionary to ensure only one update per anonymous player) + players_to_update = {} + for match_info in matches: + anon_player = match_info['anonymous_player'] + best_match = match_info['best_match'] + rank = anon_player['rank'] + + if best_match['confidence'] >= confidence_threshold and rank not in players_to_update: + # This match has high enough confidence to auto-apply + # Only add if we haven't already found a match for this rank + players_to_update[rank] = { + 'anonymous_player': anon_player, + 'match': best_match + } + + if not players_to_update: + self.stdout.write("No players met the confidence threshold for auto-matching.") + return 0 # Return 0 because no players were updated + + self.stdout.write(f"Found {len(players_to_update)} players to update.") + + # Process the file line by line + updated_count = 0 + updated_lines = [] + already_updated_ranks = set() # Track which ranks we've already updated + + # First, we need to find the data start line + data_start_line = 0 + for i, line in enumerate(lines): + if ';RANG;NOM;PRENOM;' in line: + data_start_line = i + 1 + break + + # Keep header lines unchanged + updated_lines.extend(lines[:data_start_line]) + + # Process data lines + for line in lines[data_start_line:]: + if not line.strip(): + updated_lines.append(line) + continue + + # Parse the line + values = line.strip().split(';') + if len(values) < 3: + updated_lines.append(line) + continue + + # Check if this is an anonymous player line + rank = values[1].strip() if len(values) > 1 else '' + name = values[2].strip() if len(values) > 2 else '' + first_name = values[3].strip() if len(values) > 3 else '' + + # Skip if we've already updated this rank (prevent duplicates) + if rank in already_updated_ranks: + updated_lines.append(line) + continue + + # CRITICAL CHECK: Only update if this is actually an anonymous player + # Check if player is anonymous (empty or missing name fields) + is_anonymous = not name or not first_name + + if rank in players_to_update and is_anonymous: + # This is an anonymous player line with a match to apply + update_info = players_to_update[rank] + matched_player = update_info['match']['player'] + + # Log the current values for debugging + self.stdout.write(f"Updating anonymous player at rank {rank}. Current values: Name='{name}', First name='{first_name}'") + + # Update this line with matched player info + + # Basic information: name and first name + values[2] = matched_player['name'] # Last name + values[3] = matched_player['first_name'] # First name + + # Update nationality if available + if matched_player['nationality'] != 'N/A' and len(values) > 4: + values[4] = matched_player['nationality'] + + # Update license if available + if matched_player['license'] != 'N/A' and len(values) > 5: + values[5] = matched_player['license'] + + # Additional fields: + + # Club code (position 10) + if matched_player['club_code'] != 'N/A' and len(values) > 10: + values[10] = matched_player['club_code'] + + # Club name (position 11) + if matched_player['club'] != 'N/A' and len(values) > 11: + values[11] = matched_player['club'] + + # Birth year (position 14) + if matched_player['birth_year'] != 'N/A' and len(values) > 14: + values[14] = matched_player['birth_year'] + + # Reconstruct the line + updated_line = ';'.join(values) + '\n' + updated_lines.append(updated_line) + updated_count += 1 + + # Mark this rank as updated to prevent duplicates + already_updated_ranks.add(rank) + + self.stdout.write(f"Updated player rank {rank}: {matched_player['name']} {matched_player['first_name']}") + else: + # Not an anonymous player or no match to apply - keep the line unchanged + updated_lines.append(line) + + # If this is a non-anonymous player with a rank that was in our update list, + # log a warning that we skipped it + if rank in players_to_update and not is_anonymous: + self.stdout.write(self.style.WARNING( + f"WARNING: Skipped rank {rank} because it already contains a non-anonymous player: {name} {first_name}" + )) + + # Write the updated file + with open(file_path, 'w', encoding='utf-8') as f: + f.writelines(updated_lines) + + self.stdout.write(self.style.SUCCESS(f"\nUpdated {updated_count} players in the rankings file.")) + self.stdout.write(f"Original file backed up to: {backup_path}") + + return updated_count # Return the count of updated players + + def iterative_match_anonymous_players(self, file_path, rankings_dir, options): + """ + Iteratively match anonymous players until no more matches can be found. + Uses temporary files to optimize processing speed. + """ + + iteration = 1 + total_matched = 0 + changes_made = True + + self.stdout.write(self.style.SUCCESS("\n=== Starting optimized iterative matching process ===")) + + # Load initial data + current_players, current_metadata = self.parse_rankings_file(file_path) + + # Count anonymous players at the start + anonymous_players = [p for p in current_players if self.is_anonymous_player(p)] + initial_anonymous_count = len(anonymous_players) + + if initial_anonymous_count == 0: + self.stdout.write(self.style.SUCCESS("No anonymous players found. Process complete!")) + return + + self.stdout.write(f"Initial anonymous players: {initial_anonymous_count}") + + # Find previous month file + prev_month_file = self.find_previous_month_file(current_metadata, rankings_dir) + if not prev_month_file: + self.stderr.write(self.style.ERROR('Previous month rankings file not found!')) + return + + self.stdout.write(f'Using previous month file: {os.path.basename(prev_month_file)}') + + # Load previous month data + prev_players, prev_metadata = self.parse_rankings_file(prev_month_file) + + # Create temp directory for our working files + with tempfile.TemporaryDirectory() as temp_dir: + self.stdout.write(f"Created temporary directory for working files: {temp_dir}") + + # Generate initial temp files + anon_file = os.path.join(temp_dir, "anonymous_players.json") + prev_players_file = os.path.join(temp_dir, "prev_month_players.json") + matches_file = os.path.join(temp_dir, "matches.json") + + # Extract anonymous players and filter previous month players + self.stdout.write("Creating initial working files...") + filtered_data = self.create_filtered_working_files(current_players, prev_players, anon_file, prev_players_file) + + anon_count = filtered_data['anon_count'] + prev_count = filtered_data['prev_count'] + self.stdout.write(f"Extracted {anon_count} anonymous players and {prev_count} eligible previous month players") + + # Main iteration loop + while changes_made and anon_count > 0: + self.stdout.write(self.style.SUCCESS(f"\n--- Iteration {iteration} ---")) + self.stdout.write(f"Anonymous players remaining: {anon_count}") + self.stdout.write(f"Previous month players to check: {prev_count}") + + # Process the current state of temp files + matched_count = self.match_players_from_temp_files( + anon_file, prev_players_file, matches_file, + file_path, current_metadata, options + ) + + # Check if changes were made + if matched_count > 0: + total_matched += matched_count + self.stdout.write(self.style.SUCCESS( + f"Iteration {iteration} complete: Matched {matched_count} players" + )) + changes_made = True + + # Update current players from the main file + current_players, _ = self.parse_rankings_file(file_path) + + # Update temp files for next iteration + filtered_data = self.create_filtered_working_files(current_players, prev_players, anon_file, prev_players_file) + anon_count = filtered_data['anon_count'] + prev_count = filtered_data['prev_count'] + + self.stdout.write(f"Updated working files: {anon_count} anonymous players and {prev_count} eligible previous month players") + else: + self.stdout.write(self.style.SUCCESS(f"Iteration {iteration} complete: No new matches found")) + changes_made = False + + # Increment iteration counter + iteration += 1 + + # Prevent infinite loops (optional safety check) + if iteration > 10: # Cap at 10 iterations maximum + self.stdout.write(self.style.WARNING("Maximum iterations reached (10). Stopping process.")) + break + + # Final summary + self.stdout.write(self.style.SUCCESS("\n=== Iterative matching process complete ===")) + self.stdout.write(f"Total iterations: {iteration - 1}") + self.stdout.write(f"Total players matched: {total_matched}") + + # Final statistics + final_players, _ = self.parse_rankings_file(file_path) + final_anonymous_count = len([p for p in final_players if self.is_anonymous_player(p)]) + self.stdout.write(f"Anonymous players remaining: {final_anonymous_count}") + + # Calculate improvement percentage + if initial_anonymous_count > 0: # Avoid division by zero + improvement = ((initial_anonymous_count - final_anonymous_count) / initial_anonymous_count) * 100 + self.stdout.write(f"Data completeness improved by {improvement:.1f}%") + + def create_filtered_working_files(self, current_players, prev_players, anon_file, prev_players_file): + """ + Create filtered working files: + 1. anonymous_players.json - Contains only anonymous players from current month + 2. prev_month_players.json - Contains only players from previous month not in current month + + Returns dictionary with counts of players in each file + """ + + # Extract anonymous players from current month + anonymous_players = [p for p in current_players if self.is_anonymous_player(p)] + + # Create lookup for current non-anonymous players + current_players_lookup = {} + for player in current_players: + if not self.is_anonymous_player(player): + # License lookup + if player['license'] != 'N/A' and player['license']: + current_players_lookup[f"license_{player['license']}"] = True + + # Filter previous month players (only keep those not in current month) + filtered_prev_players = [] + for player in prev_players: + if self.is_anonymous_player(player): + continue # Skip anonymous players from previous month + + # Check if this player exists in current month + exists_in_current = False + + # Check by license + if player['license'] != 'N/A' and player['license']: + if f"license_{player['license']}" in current_players_lookup: + exists_in_current = True + + # Add to filtered list if not in current month + if not exists_in_current: + filtered_prev_players.append(player) + + # Write anonymous players to file + with open(anon_file, 'w', encoding='utf-8') as f: + json.dump(anonymous_players, f, ensure_ascii=False) + + # Write filtered previous players to file + with open(prev_players_file, 'w', encoding='utf-8') as f: + json.dump(filtered_prev_players, f, ensure_ascii=False) + + return { + 'anon_count': len(anonymous_players), + 'prev_count': len(filtered_prev_players) + } + + def match_players_from_temp_files(self, anon_file, prev_players_file, matches_file, + original_file, current_metadata, options): + """ + Match players between the anonymous and previous month temp files + and update the original file with matches + """ + + # Load anonymous players + with open(anon_file, 'r', encoding='utf-8') as f: + anonymous_players = json.load(f) + + # Load previous month players + with open(prev_players_file, 'r', encoding='utf-8') as f: + prev_players = json.load(f) + + if not anonymous_players or not prev_players: + return 0 + + # Create indexes for efficient lookup + current_players_indexes = { + 'license_index': {}, + 'name_index': {}, + 'name_club_league_index': {} + } + + # Find matches + results = [] + + for anon_player in anonymous_players: + potential_matches = self.find_potential_matches(anon_player, prev_players, current_players_indexes, options) + if potential_matches: + best_match = potential_matches[0] # Highest confidence match + + # Record the match info + match_info = { + 'anonymous_player': anon_player, + 'potential_matches': potential_matches, + 'best_match': best_match + } + results.append(match_info) + + # Save matches to file + with open(matches_file, 'w', encoding='utf-8') as f: + # We can't directly serialize the complex match data, so extract key info + serializable_results = [] + for match_info in results: + serializable_results.append({ + 'anonymous_player': match_info['anonymous_player'], + 'best_match': { + 'player': match_info['best_match']['player'], + 'confidence': match_info['best_match']['confidence'], + 'match_reasons': match_info['best_match']['match_reasons'] + } + }) + json.dump(serializable_results, f, ensure_ascii=False) + + # Apply matches to the original file + if results: + matched_count = self.update_rankings_with_matches( + original_file, anonymous_players, results, options['confidence'], options + ) + return matched_count + + return 0 diff --git a/tournaments/templates/admin/tournaments/index.html b/tournaments/templates/admin/tournaments/index.html new file mode 100644 index 0000000..2676ff4 --- /dev/null +++ b/tournaments/templates/admin/tournaments/index.html @@ -0,0 +1,13 @@ +{% extends "admin/index.html" %} + +{% block content %} +
+

๐Ÿ› ๏ธ Debug Tools

+

Quick access to development and debugging utilities:

+ + ๐Ÿ“ Debug Tools Dashboard + +
+ +{{ block.super }} +{% endblock %}