You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
222 lines
8.6 KiB
222 lines
8.6 KiB
from django.core.management.base import BaseCommand
|
|
from datetime import datetime, timedelta
|
|
import logging
|
|
|
|
class Command(BaseCommand):
|
|
help = 'Test FFT all tournaments scraping with various filters'
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument(
|
|
'--sorting',
|
|
type=str,
|
|
default='dateDebut+asc',
|
|
choices=['dateDebut+asc', 'dateDebut+desc', '_DISTANCE_'],
|
|
help='Sorting option (default: dateDebut+asc)'
|
|
)
|
|
parser.add_argument(
|
|
'--page',
|
|
type=int,
|
|
default=0,
|
|
help='Page number to scrape (default: 0)'
|
|
)
|
|
parser.add_argument(
|
|
'--city',
|
|
type=str,
|
|
default='',
|
|
help='City to search around'
|
|
)
|
|
parser.add_argument(
|
|
'--distance',
|
|
type=float,
|
|
default=15.0,
|
|
help='Distance in km (default: 15)'
|
|
)
|
|
parser.add_argument(
|
|
'--categories',
|
|
nargs='*',
|
|
default=[],
|
|
help='Tournament categories to filter by'
|
|
)
|
|
parser.add_argument(
|
|
'--levels',
|
|
nargs='*',
|
|
default=[],
|
|
help='Tournament levels to filter by'
|
|
)
|
|
parser.add_argument(
|
|
'--ages',
|
|
nargs='*',
|
|
default=[],
|
|
help='Age categories to filter by'
|
|
)
|
|
parser.add_argument(
|
|
'--types',
|
|
nargs='*',
|
|
default=[],
|
|
help='Tournament types to filter by'
|
|
)
|
|
parser.add_argument(
|
|
'--national-cup',
|
|
action='store_true',
|
|
help='Filter for national cup tournaments only'
|
|
)
|
|
parser.add_argument(
|
|
'--lat',
|
|
type=float,
|
|
help='Latitude for location-based search'
|
|
)
|
|
parser.add_argument(
|
|
'--lng',
|
|
type=float,
|
|
help='Longitude for location-based search'
|
|
)
|
|
parser.add_argument(
|
|
'--days-ahead',
|
|
type=int,
|
|
default=90,
|
|
help='How many days ahead to search (default: 90)'
|
|
)
|
|
parser.add_argument(
|
|
'--start-date',
|
|
type=str,
|
|
help='Start date in DD/MM/YY format (overrides --days-ahead)'
|
|
)
|
|
parser.add_argument(
|
|
'--end-date',
|
|
type=str,
|
|
help='End date in DD/MM/YY format (overrides --days-ahead)'
|
|
)
|
|
parser.add_argument(
|
|
'--verbose',
|
|
action='store_true',
|
|
help='Enable verbose logging'
|
|
)
|
|
|
|
def handle(self, *args, **options):
|
|
if options['verbose']:
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
# Extract options
|
|
sorting_option = options['sorting']
|
|
page = options['page']
|
|
city = options['city']
|
|
distance = options['distance']
|
|
categories = options['categories']
|
|
levels = options['levels']
|
|
ages = options['ages']
|
|
tournament_types = options['types']
|
|
national_cup = options['national_cup']
|
|
lat = options['lat']
|
|
lng = options['lng']
|
|
verbose = options['verbose']
|
|
|
|
# Calculate date range
|
|
if options['start_date'] and options['end_date']:
|
|
start_date_str = options['start_date']
|
|
end_date_str = options['end_date']
|
|
else:
|
|
start_date = datetime.now()
|
|
end_date = start_date + timedelta(days=options['days_ahead'])
|
|
start_date_str = start_date.strftime('%d/%m/%y')
|
|
end_date_str = end_date.strftime('%d/%m/%y')
|
|
|
|
self.stdout.write(self.style.SUCCESS("=== FFT All Tournaments Scraper ==="))
|
|
self.stdout.write(f"Sorting: {sorting_option}")
|
|
self.stdout.write(f"Page: {page}")
|
|
self.stdout.write(f"Date range: {start_date_str} to {end_date_str}")
|
|
self.stdout.write(f"City: {city if city else 'Not specified'}")
|
|
self.stdout.write(f"Distance: {distance} km")
|
|
self.stdout.write(f"Categories: {categories if categories else 'All'}")
|
|
self.stdout.write(f"Levels: {levels if levels else 'All'}")
|
|
self.stdout.write(f"Ages: {ages if ages else 'All'}")
|
|
self.stdout.write(f"Types: {tournament_types if tournament_types else 'All'}")
|
|
self.stdout.write(f"National Cup: {'Yes' if national_cup else 'No'}")
|
|
if lat and lng:
|
|
self.stdout.write(f"Location: {lat}, {lng}")
|
|
self.stdout.write(f"Method: Playwright (Chrome-free)")
|
|
self.stdout.write("")
|
|
|
|
try:
|
|
from api.utils import scrape_fft_all_tournaments
|
|
self.stdout.write("🚀 Testing general tournament scraping...")
|
|
|
|
result = scrape_fft_all_tournaments(
|
|
sorting_option=sorting_option,
|
|
page=page,
|
|
start_date=start_date_str,
|
|
end_date=end_date_str,
|
|
city=city,
|
|
distance=distance,
|
|
categories=categories,
|
|
levels=levels,
|
|
lat=lat,
|
|
lng=lng,
|
|
ages=ages,
|
|
tournament_types=tournament_types,
|
|
national_cup=national_cup
|
|
)
|
|
|
|
# Debug: Show what we got (only in verbose mode)
|
|
if verbose:
|
|
self.stdout.write(f"🔍 Raw result: {result}")
|
|
|
|
if result:
|
|
tournaments = result.get('tournaments', [])
|
|
self.stdout.write(self.style.SUCCESS(f"✅ SUCCESS: {len(tournaments)} tournaments found"))
|
|
|
|
if tournaments:
|
|
self.stdout.write("\n📝 Sample tournaments:")
|
|
# Show first 3 tournaments
|
|
for i, tournament in enumerate(tournaments[:3]):
|
|
self.stdout.write(f"\n Tournament {i+1}:")
|
|
self.stdout.write(f" ID: {tournament.get('id')}")
|
|
self.stdout.write(f" Name: {tournament.get('libelle')}")
|
|
self.stdout.write(f" Date: {tournament.get('dateDebut', {}).get('date', 'N/A')}")
|
|
self.stdout.write(f" Club: {tournament.get('nomClub', 'N/A')}")
|
|
self.stdout.write(f" City: {tournament.get('villeEngagement', 'N/A')}")
|
|
self.stdout.write(f" Category: {tournament.get('categorieTournoi', 'N/A')}")
|
|
self.stdout.write(f" Type: {tournament.get('type', 'N/A')}")
|
|
if tournament.get('jugeArbitre'):
|
|
self.stdout.write(f" Judge: {tournament.get('jugeArbitre', {}).get('nom', 'N/A')}")
|
|
|
|
self.stdout.write(f"\n📊 Summary:")
|
|
self.stdout.write(f" Total tournaments: {len(tournaments)}")
|
|
self.stdout.write(f" Current page: {page}")
|
|
self.stdout.write(f" Total results available: {result.get('total_results', 'Unknown')}")
|
|
|
|
# Analysis of results
|
|
if tournaments:
|
|
cities = set()
|
|
clubs = set()
|
|
categories = set()
|
|
types = set()
|
|
|
|
for tournament in tournaments:
|
|
if tournament.get('villeEngagement'):
|
|
cities.add(tournament['villeEngagement'])
|
|
if tournament.get('nomClub'):
|
|
clubs.add(tournament['nomClub'])
|
|
if tournament.get('categorieTournoi'):
|
|
categories.add(tournament['categorieTournoi'])
|
|
if tournament.get('type'):
|
|
types.add(tournament['type'])
|
|
|
|
self.stdout.write(f"\n🔍 Analysis:")
|
|
self.stdout.write(f" Unique cities: {len(cities)}")
|
|
self.stdout.write(f" Unique clubs: {len(clubs)}")
|
|
self.stdout.write(f" Unique categories: {len(categories)}")
|
|
self.stdout.write(f" Unique types: {len(types)}")
|
|
|
|
if verbose:
|
|
self.stdout.write(f"\n Cities: {sorted(list(cities))[:10]}") # Show first 10
|
|
self.stdout.write(f" Categories: {sorted(list(categories))}")
|
|
self.stdout.write(f" Types: {sorted(list(types))}")
|
|
|
|
else:
|
|
self.stdout.write(self.style.ERROR("❌ FAILED: No tournaments found"))
|
|
|
|
except Exception as e:
|
|
self.stdout.write(self.style.ERROR(f"❌ ERROR: {e}"))
|
|
import traceback
|
|
if verbose:
|
|
self.stdout.write(traceback.format_exc())
|
|
|