parent
590a652e83
commit
576bc2f273
@ -0,0 +1,222 @@ |
|||||||
|
from django.core.management.base import BaseCommand |
||||||
|
from datetime import datetime, timedelta |
||||||
|
import logging |
||||||
|
|
||||||
|
class Command(BaseCommand): |
||||||
|
help = 'Test FFT all tournaments scraping with various filters' |
||||||
|
|
||||||
|
def add_arguments(self, parser): |
||||||
|
parser.add_argument( |
||||||
|
'--sorting', |
||||||
|
type=str, |
||||||
|
default='dateDebut+asc', |
||||||
|
choices=['dateDebut+asc', 'dateDebut+desc', '_DISTANCE_'], |
||||||
|
help='Sorting option (default: dateDebut+asc)' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--page', |
||||||
|
type=int, |
||||||
|
default=0, |
||||||
|
help='Page number to scrape (default: 0)' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--city', |
||||||
|
type=str, |
||||||
|
default='', |
||||||
|
help='City to search around' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--distance', |
||||||
|
type=float, |
||||||
|
default=15.0, |
||||||
|
help='Distance in km (default: 15)' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--categories', |
||||||
|
nargs='*', |
||||||
|
default=[], |
||||||
|
help='Tournament categories to filter by' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--levels', |
||||||
|
nargs='*', |
||||||
|
default=[], |
||||||
|
help='Tournament levels to filter by' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--ages', |
||||||
|
nargs='*', |
||||||
|
default=[], |
||||||
|
help='Age categories to filter by' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--types', |
||||||
|
nargs='*', |
||||||
|
default=[], |
||||||
|
help='Tournament types to filter by' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--national-cup', |
||||||
|
action='store_true', |
||||||
|
help='Filter for national cup tournaments only' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--lat', |
||||||
|
type=float, |
||||||
|
help='Latitude for location-based search' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--lng', |
||||||
|
type=float, |
||||||
|
help='Longitude for location-based search' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--days-ahead', |
||||||
|
type=int, |
||||||
|
default=90, |
||||||
|
help='How many days ahead to search (default: 90)' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--start-date', |
||||||
|
type=str, |
||||||
|
help='Start date in DD/MM/YY format (overrides --days-ahead)' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--end-date', |
||||||
|
type=str, |
||||||
|
help='End date in DD/MM/YY format (overrides --days-ahead)' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--verbose', |
||||||
|
action='store_true', |
||||||
|
help='Enable verbose logging' |
||||||
|
) |
||||||
|
|
||||||
|
def handle(self, *args, **options): |
||||||
|
if options['verbose']: |
||||||
|
logging.basicConfig(level=logging.INFO) |
||||||
|
|
||||||
|
# Extract options |
||||||
|
sorting_option = options['sorting'] |
||||||
|
page = options['page'] |
||||||
|
city = options['city'] |
||||||
|
distance = options['distance'] |
||||||
|
categories = options['categories'] |
||||||
|
levels = options['levels'] |
||||||
|
ages = options['ages'] |
||||||
|
tournament_types = options['types'] |
||||||
|
national_cup = options['national_cup'] |
||||||
|
lat = options['lat'] |
||||||
|
lng = options['lng'] |
||||||
|
verbose = options['verbose'] |
||||||
|
|
||||||
|
# Calculate date range |
||||||
|
if options['start_date'] and options['end_date']: |
||||||
|
start_date_str = options['start_date'] |
||||||
|
end_date_str = options['end_date'] |
||||||
|
else: |
||||||
|
start_date = datetime.now() |
||||||
|
end_date = start_date + timedelta(days=options['days_ahead']) |
||||||
|
start_date_str = start_date.strftime('%d/%m/%y') |
||||||
|
end_date_str = end_date.strftime('%d/%m/%y') |
||||||
|
|
||||||
|
self.stdout.write(self.style.SUCCESS("=== FFT All Tournaments Scraper ===")) |
||||||
|
self.stdout.write(f"Sorting: {sorting_option}") |
||||||
|
self.stdout.write(f"Page: {page}") |
||||||
|
self.stdout.write(f"Date range: {start_date_str} to {end_date_str}") |
||||||
|
self.stdout.write(f"City: {city if city else 'Not specified'}") |
||||||
|
self.stdout.write(f"Distance: {distance} km") |
||||||
|
self.stdout.write(f"Categories: {categories if categories else 'All'}") |
||||||
|
self.stdout.write(f"Levels: {levels if levels else 'All'}") |
||||||
|
self.stdout.write(f"Ages: {ages if ages else 'All'}") |
||||||
|
self.stdout.write(f"Types: {tournament_types if tournament_types else 'All'}") |
||||||
|
self.stdout.write(f"National Cup: {'Yes' if national_cup else 'No'}") |
||||||
|
if lat and lng: |
||||||
|
self.stdout.write(f"Location: {lat}, {lng}") |
||||||
|
self.stdout.write(f"Method: Playwright (Chrome-free)") |
||||||
|
self.stdout.write("") |
||||||
|
|
||||||
|
try: |
||||||
|
from api.utils import scrape_fft_all_tournaments |
||||||
|
self.stdout.write("🚀 Testing general tournament scraping...") |
||||||
|
|
||||||
|
result = scrape_fft_all_tournaments( |
||||||
|
sorting_option=sorting_option, |
||||||
|
page=page, |
||||||
|
start_date=start_date_str, |
||||||
|
end_date=end_date_str, |
||||||
|
city=city, |
||||||
|
distance=distance, |
||||||
|
categories=categories, |
||||||
|
levels=levels, |
||||||
|
lat=lat, |
||||||
|
lng=lng, |
||||||
|
ages=ages, |
||||||
|
tournament_types=tournament_types, |
||||||
|
national_cup=national_cup |
||||||
|
) |
||||||
|
|
||||||
|
# Debug: Show what we got (only in verbose mode) |
||||||
|
if verbose: |
||||||
|
self.stdout.write(f"🔍 Raw result: {result}") |
||||||
|
|
||||||
|
if result: |
||||||
|
tournaments = result.get('tournaments', []) |
||||||
|
self.stdout.write(self.style.SUCCESS(f"✅ SUCCESS: {len(tournaments)} tournaments found")) |
||||||
|
|
||||||
|
if tournaments: |
||||||
|
self.stdout.write("\n📝 Sample tournaments:") |
||||||
|
# Show first 3 tournaments |
||||||
|
for i, tournament in enumerate(tournaments[:3]): |
||||||
|
self.stdout.write(f"\n Tournament {i+1}:") |
||||||
|
self.stdout.write(f" ID: {tournament.get('id')}") |
||||||
|
self.stdout.write(f" Name: {tournament.get('libelle')}") |
||||||
|
self.stdout.write(f" Date: {tournament.get('dateDebut', {}).get('date', 'N/A')}") |
||||||
|
self.stdout.write(f" Club: {tournament.get('nomClub', 'N/A')}") |
||||||
|
self.stdout.write(f" City: {tournament.get('villeEngagement', 'N/A')}") |
||||||
|
self.stdout.write(f" Category: {tournament.get('categorieTournoi', 'N/A')}") |
||||||
|
self.stdout.write(f" Type: {tournament.get('type', 'N/A')}") |
||||||
|
if tournament.get('jugeArbitre'): |
||||||
|
self.stdout.write(f" Judge: {tournament.get('jugeArbitre', {}).get('nom', 'N/A')}") |
||||||
|
|
||||||
|
self.stdout.write(f"\n📊 Summary:") |
||||||
|
self.stdout.write(f" Total tournaments: {len(tournaments)}") |
||||||
|
self.stdout.write(f" Current page: {page}") |
||||||
|
self.stdout.write(f" Total results available: {result.get('total_results', 'Unknown')}") |
||||||
|
|
||||||
|
# Analysis of results |
||||||
|
if tournaments: |
||||||
|
cities = set() |
||||||
|
clubs = set() |
||||||
|
categories = set() |
||||||
|
types = set() |
||||||
|
|
||||||
|
for tournament in tournaments: |
||||||
|
if tournament.get('villeEngagement'): |
||||||
|
cities.add(tournament['villeEngagement']) |
||||||
|
if tournament.get('nomClub'): |
||||||
|
clubs.add(tournament['nomClub']) |
||||||
|
if tournament.get('categorieTournoi'): |
||||||
|
categories.add(tournament['categorieTournoi']) |
||||||
|
if tournament.get('type'): |
||||||
|
types.add(tournament['type']) |
||||||
|
|
||||||
|
self.stdout.write(f"\n🔍 Analysis:") |
||||||
|
self.stdout.write(f" Unique cities: {len(cities)}") |
||||||
|
self.stdout.write(f" Unique clubs: {len(clubs)}") |
||||||
|
self.stdout.write(f" Unique categories: {len(categories)}") |
||||||
|
self.stdout.write(f" Unique types: {len(types)}") |
||||||
|
|
||||||
|
if verbose: |
||||||
|
self.stdout.write(f"\n Cities: {sorted(list(cities))[:10]}") # Show first 10 |
||||||
|
self.stdout.write(f" Categories: {sorted(list(categories))}") |
||||||
|
self.stdout.write(f" Types: {sorted(list(types))}") |
||||||
|
|
||||||
|
else: |
||||||
|
self.stdout.write(self.style.ERROR("❌ FAILED: No tournaments found")) |
||||||
|
|
||||||
|
except Exception as e: |
||||||
|
self.stdout.write(self.style.ERROR(f"❌ ERROR: {e}")) |
||||||
|
import traceback |
||||||
|
if verbose: |
||||||
|
self.stdout.write(traceback.format_exc()) |
||||||
@ -0,0 +1,103 @@ |
|||||||
|
from django.core.management.base import BaseCommand |
||||||
|
from datetime import datetime, timedelta |
||||||
|
import logging |
||||||
|
|
||||||
|
class Command(BaseCommand): |
||||||
|
help = 'Test FFT tournament scraping with Playwright' |
||||||
|
|
||||||
|
def add_arguments(self, parser): |
||||||
|
parser.add_argument( |
||||||
|
'--club-code', |
||||||
|
type=str, |
||||||
|
default='62130180', |
||||||
|
help='Club code for testing (default: 62130180)' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--club-name', |
||||||
|
type=str, |
||||||
|
default='TENNIS SPORTING CLUB DE CASSIS', |
||||||
|
help='Club name for testing' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--all-pages', |
||||||
|
action='store_true', |
||||||
|
help='Test all pages scraping' |
||||||
|
) |
||||||
|
parser.add_argument( |
||||||
|
'--verbose', |
||||||
|
action='store_true', |
||||||
|
help='Enable verbose logging' |
||||||
|
) |
||||||
|
|
||||||
|
def handle(self, *args, **options): |
||||||
|
if options['verbose']: |
||||||
|
logging.basicConfig(level=logging.INFO) |
||||||
|
|
||||||
|
club_code = options['club_code'] |
||||||
|
club_name = options['club_name'] |
||||||
|
all_pages = options['all_pages'] |
||||||
|
verbose = options['verbose'] |
||||||
|
|
||||||
|
# Calculate date range |
||||||
|
start_date = datetime.now() |
||||||
|
end_date = start_date + timedelta(days=90) |
||||||
|
start_date_str = start_date.strftime('%d/%m/%y') |
||||||
|
end_date_str = end_date.strftime('%d/%m/%y') |
||||||
|
|
||||||
|
self.stdout.write(self.style.SUCCESS("=== FFT Tournament Scraper ===")) |
||||||
|
self.stdout.write(f"Club: {club_name} ({club_code})") |
||||||
|
self.stdout.write(f"Date range: {start_date_str} to {end_date_str}") |
||||||
|
self.stdout.write(f"Method: Playwright (Chrome-free)") |
||||||
|
self.stdout.write("") |
||||||
|
|
||||||
|
try: |
||||||
|
if all_pages: |
||||||
|
from api.utils import scrape_fft_club_tournaments_all_pages |
||||||
|
self.stdout.write("🚀 Testing complete tournament scraping...") |
||||||
|
|
||||||
|
result = scrape_fft_club_tournaments_all_pages( |
||||||
|
club_code=club_code, |
||||||
|
club_name=club_name, |
||||||
|
start_date=start_date_str, |
||||||
|
end_date=end_date_str |
||||||
|
) |
||||||
|
else: |
||||||
|
from api.utils import scrape_fft_club_tournaments |
||||||
|
self.stdout.write("🚀 Testing single page scraping...") |
||||||
|
|
||||||
|
result = scrape_fft_club_tournaments( |
||||||
|
club_code=club_code, |
||||||
|
club_name=club_name, |
||||||
|
start_date=start_date_str, |
||||||
|
end_date=end_date_str, |
||||||
|
page=0 |
||||||
|
) |
||||||
|
|
||||||
|
# Debug: Show what we got (only in verbose mode) |
||||||
|
if verbose: |
||||||
|
self.stdout.write(f"🔍 Raw result: {result}") |
||||||
|
|
||||||
|
if result: |
||||||
|
tournaments = result.get('tournaments', []) |
||||||
|
self.stdout.write(self.style.SUCCESS(f"✅ SUCCESS: {len(tournaments)} tournaments found")) |
||||||
|
|
||||||
|
if tournaments: |
||||||
|
self.stdout.write("\n📝 Sample tournament:") |
||||||
|
sample = tournaments[0] |
||||||
|
self.stdout.write(f" ID: {sample.get('id')}") |
||||||
|
self.stdout.write(f" Name: {sample.get('libelle')}") |
||||||
|
self.stdout.write(f" Date: {sample.get('dateDebut', {}).get('date', 'N/A')}") |
||||||
|
self.stdout.write(f" Judge: {sample.get('jugeArbitre', {}).get('nom', 'N/A')}") |
||||||
|
|
||||||
|
self.stdout.write(f"\n📊 Summary:") |
||||||
|
self.stdout.write(f" Total tournaments: {len(tournaments)}") |
||||||
|
self.stdout.write(f" Pages scraped: {result.get('pages_scraped', 1)}") |
||||||
|
|
||||||
|
else: |
||||||
|
self.stdout.write(self.style.ERROR("❌ FAILED: No tournaments found")) |
||||||
|
|
||||||
|
except Exception as e: |
||||||
|
self.stdout.write(self.style.ERROR(f"❌ ERROR: {e}")) |
||||||
|
import traceback |
||||||
|
if verbose: |
||||||
|
self.stdout.write(traceback.format_exc()) |
||||||
Loading…
Reference in new issue