Razmig Sarkissian
4 months ago
parent
590a652e83
commit
576bc2f273
@ -0,0 +1,222 @@ |
||||
from django.core.management.base import BaseCommand |
||||
from datetime import datetime, timedelta |
||||
import logging |
||||
|
||||
class Command(BaseCommand): |
||||
help = 'Test FFT all tournaments scraping with various filters' |
||||
|
||||
def add_arguments(self, parser): |
||||
parser.add_argument( |
||||
'--sorting', |
||||
type=str, |
||||
default='dateDebut+asc', |
||||
choices=['dateDebut+asc', 'dateDebut+desc', '_DISTANCE_'], |
||||
help='Sorting option (default: dateDebut+asc)' |
||||
) |
||||
parser.add_argument( |
||||
'--page', |
||||
type=int, |
||||
default=0, |
||||
help='Page number to scrape (default: 0)' |
||||
) |
||||
parser.add_argument( |
||||
'--city', |
||||
type=str, |
||||
default='', |
||||
help='City to search around' |
||||
) |
||||
parser.add_argument( |
||||
'--distance', |
||||
type=float, |
||||
default=15.0, |
||||
help='Distance in km (default: 15)' |
||||
) |
||||
parser.add_argument( |
||||
'--categories', |
||||
nargs='*', |
||||
default=[], |
||||
help='Tournament categories to filter by' |
||||
) |
||||
parser.add_argument( |
||||
'--levels', |
||||
nargs='*', |
||||
default=[], |
||||
help='Tournament levels to filter by' |
||||
) |
||||
parser.add_argument( |
||||
'--ages', |
||||
nargs='*', |
||||
default=[], |
||||
help='Age categories to filter by' |
||||
) |
||||
parser.add_argument( |
||||
'--types', |
||||
nargs='*', |
||||
default=[], |
||||
help='Tournament types to filter by' |
||||
) |
||||
parser.add_argument( |
||||
'--national-cup', |
||||
action='store_true', |
||||
help='Filter for national cup tournaments only' |
||||
) |
||||
parser.add_argument( |
||||
'--lat', |
||||
type=float, |
||||
help='Latitude for location-based search' |
||||
) |
||||
parser.add_argument( |
||||
'--lng', |
||||
type=float, |
||||
help='Longitude for location-based search' |
||||
) |
||||
parser.add_argument( |
||||
'--days-ahead', |
||||
type=int, |
||||
default=90, |
||||
help='How many days ahead to search (default: 90)' |
||||
) |
||||
parser.add_argument( |
||||
'--start-date', |
||||
type=str, |
||||
help='Start date in DD/MM/YY format (overrides --days-ahead)' |
||||
) |
||||
parser.add_argument( |
||||
'--end-date', |
||||
type=str, |
||||
help='End date in DD/MM/YY format (overrides --days-ahead)' |
||||
) |
||||
parser.add_argument( |
||||
'--verbose', |
||||
action='store_true', |
||||
help='Enable verbose logging' |
||||
) |
||||
|
||||
def handle(self, *args, **options): |
||||
if options['verbose']: |
||||
logging.basicConfig(level=logging.INFO) |
||||
|
||||
# Extract options |
||||
sorting_option = options['sorting'] |
||||
page = options['page'] |
||||
city = options['city'] |
||||
distance = options['distance'] |
||||
categories = options['categories'] |
||||
levels = options['levels'] |
||||
ages = options['ages'] |
||||
tournament_types = options['types'] |
||||
national_cup = options['national_cup'] |
||||
lat = options['lat'] |
||||
lng = options['lng'] |
||||
verbose = options['verbose'] |
||||
|
||||
# Calculate date range |
||||
if options['start_date'] and options['end_date']: |
||||
start_date_str = options['start_date'] |
||||
end_date_str = options['end_date'] |
||||
else: |
||||
start_date = datetime.now() |
||||
end_date = start_date + timedelta(days=options['days_ahead']) |
||||
start_date_str = start_date.strftime('%d/%m/%y') |
||||
end_date_str = end_date.strftime('%d/%m/%y') |
||||
|
||||
self.stdout.write(self.style.SUCCESS("=== FFT All Tournaments Scraper ===")) |
||||
self.stdout.write(f"Sorting: {sorting_option}") |
||||
self.stdout.write(f"Page: {page}") |
||||
self.stdout.write(f"Date range: {start_date_str} to {end_date_str}") |
||||
self.stdout.write(f"City: {city if city else 'Not specified'}") |
||||
self.stdout.write(f"Distance: {distance} km") |
||||
self.stdout.write(f"Categories: {categories if categories else 'All'}") |
||||
self.stdout.write(f"Levels: {levels if levels else 'All'}") |
||||
self.stdout.write(f"Ages: {ages if ages else 'All'}") |
||||
self.stdout.write(f"Types: {tournament_types if tournament_types else 'All'}") |
||||
self.stdout.write(f"National Cup: {'Yes' if national_cup else 'No'}") |
||||
if lat and lng: |
||||
self.stdout.write(f"Location: {lat}, {lng}") |
||||
self.stdout.write(f"Method: Playwright (Chrome-free)") |
||||
self.stdout.write("") |
||||
|
||||
try: |
||||
from api.utils import scrape_fft_all_tournaments |
||||
self.stdout.write("🚀 Testing general tournament scraping...") |
||||
|
||||
result = scrape_fft_all_tournaments( |
||||
sorting_option=sorting_option, |
||||
page=page, |
||||
start_date=start_date_str, |
||||
end_date=end_date_str, |
||||
city=city, |
||||
distance=distance, |
||||
categories=categories, |
||||
levels=levels, |
||||
lat=lat, |
||||
lng=lng, |
||||
ages=ages, |
||||
tournament_types=tournament_types, |
||||
national_cup=national_cup |
||||
) |
||||
|
||||
# Debug: Show what we got (only in verbose mode) |
||||
if verbose: |
||||
self.stdout.write(f"🔍 Raw result: {result}") |
||||
|
||||
if result: |
||||
tournaments = result.get('tournaments', []) |
||||
self.stdout.write(self.style.SUCCESS(f"✅ SUCCESS: {len(tournaments)} tournaments found")) |
||||
|
||||
if tournaments: |
||||
self.stdout.write("\n📝 Sample tournaments:") |
||||
# Show first 3 tournaments |
||||
for i, tournament in enumerate(tournaments[:3]): |
||||
self.stdout.write(f"\n Tournament {i+1}:") |
||||
self.stdout.write(f" ID: {tournament.get('id')}") |
||||
self.stdout.write(f" Name: {tournament.get('libelle')}") |
||||
self.stdout.write(f" Date: {tournament.get('dateDebut', {}).get('date', 'N/A')}") |
||||
self.stdout.write(f" Club: {tournament.get('nomClub', 'N/A')}") |
||||
self.stdout.write(f" City: {tournament.get('villeEngagement', 'N/A')}") |
||||
self.stdout.write(f" Category: {tournament.get('categorieTournoi', 'N/A')}") |
||||
self.stdout.write(f" Type: {tournament.get('type', 'N/A')}") |
||||
if tournament.get('jugeArbitre'): |
||||
self.stdout.write(f" Judge: {tournament.get('jugeArbitre', {}).get('nom', 'N/A')}") |
||||
|
||||
self.stdout.write(f"\n📊 Summary:") |
||||
self.stdout.write(f" Total tournaments: {len(tournaments)}") |
||||
self.stdout.write(f" Current page: {page}") |
||||
self.stdout.write(f" Total results available: {result.get('total_results', 'Unknown')}") |
||||
|
||||
# Analysis of results |
||||
if tournaments: |
||||
cities = set() |
||||
clubs = set() |
||||
categories = set() |
||||
types = set() |
||||
|
||||
for tournament in tournaments: |
||||
if tournament.get('villeEngagement'): |
||||
cities.add(tournament['villeEngagement']) |
||||
if tournament.get('nomClub'): |
||||
clubs.add(tournament['nomClub']) |
||||
if tournament.get('categorieTournoi'): |
||||
categories.add(tournament['categorieTournoi']) |
||||
if tournament.get('type'): |
||||
types.add(tournament['type']) |
||||
|
||||
self.stdout.write(f"\n🔍 Analysis:") |
||||
self.stdout.write(f" Unique cities: {len(cities)}") |
||||
self.stdout.write(f" Unique clubs: {len(clubs)}") |
||||
self.stdout.write(f" Unique categories: {len(categories)}") |
||||
self.stdout.write(f" Unique types: {len(types)}") |
||||
|
||||
if verbose: |
||||
self.stdout.write(f"\n Cities: {sorted(list(cities))[:10]}") # Show first 10 |
||||
self.stdout.write(f" Categories: {sorted(list(categories))}") |
||||
self.stdout.write(f" Types: {sorted(list(types))}") |
||||
|
||||
else: |
||||
self.stdout.write(self.style.ERROR("❌ FAILED: No tournaments found")) |
||||
|
||||
except Exception as e: |
||||
self.stdout.write(self.style.ERROR(f"❌ ERROR: {e}")) |
||||
import traceback |
||||
if verbose: |
||||
self.stdout.write(traceback.format_exc()) |
||||
@ -0,0 +1,103 @@ |
||||
from django.core.management.base import BaseCommand |
||||
from datetime import datetime, timedelta |
||||
import logging |
||||
|
||||
class Command(BaseCommand): |
||||
help = 'Test FFT tournament scraping with Playwright' |
||||
|
||||
def add_arguments(self, parser): |
||||
parser.add_argument( |
||||
'--club-code', |
||||
type=str, |
||||
default='62130180', |
||||
help='Club code for testing (default: 62130180)' |
||||
) |
||||
parser.add_argument( |
||||
'--club-name', |
||||
type=str, |
||||
default='TENNIS SPORTING CLUB DE CASSIS', |
||||
help='Club name for testing' |
||||
) |
||||
parser.add_argument( |
||||
'--all-pages', |
||||
action='store_true', |
||||
help='Test all pages scraping' |
||||
) |
||||
parser.add_argument( |
||||
'--verbose', |
||||
action='store_true', |
||||
help='Enable verbose logging' |
||||
) |
||||
|
||||
def handle(self, *args, **options): |
||||
if options['verbose']: |
||||
logging.basicConfig(level=logging.INFO) |
||||
|
||||
club_code = options['club_code'] |
||||
club_name = options['club_name'] |
||||
all_pages = options['all_pages'] |
||||
verbose = options['verbose'] |
||||
|
||||
# Calculate date range |
||||
start_date = datetime.now() |
||||
end_date = start_date + timedelta(days=90) |
||||
start_date_str = start_date.strftime('%d/%m/%y') |
||||
end_date_str = end_date.strftime('%d/%m/%y') |
||||
|
||||
self.stdout.write(self.style.SUCCESS("=== FFT Tournament Scraper ===")) |
||||
self.stdout.write(f"Club: {club_name} ({club_code})") |
||||
self.stdout.write(f"Date range: {start_date_str} to {end_date_str}") |
||||
self.stdout.write(f"Method: Playwright (Chrome-free)") |
||||
self.stdout.write("") |
||||
|
||||
try: |
||||
if all_pages: |
||||
from api.utils import scrape_fft_club_tournaments_all_pages |
||||
self.stdout.write("🚀 Testing complete tournament scraping...") |
||||
|
||||
result = scrape_fft_club_tournaments_all_pages( |
||||
club_code=club_code, |
||||
club_name=club_name, |
||||
start_date=start_date_str, |
||||
end_date=end_date_str |
||||
) |
||||
else: |
||||
from api.utils import scrape_fft_club_tournaments |
||||
self.stdout.write("🚀 Testing single page scraping...") |
||||
|
||||
result = scrape_fft_club_tournaments( |
||||
club_code=club_code, |
||||
club_name=club_name, |
||||
start_date=start_date_str, |
||||
end_date=end_date_str, |
||||
page=0 |
||||
) |
||||
|
||||
# Debug: Show what we got (only in verbose mode) |
||||
if verbose: |
||||
self.stdout.write(f"🔍 Raw result: {result}") |
||||
|
||||
if result: |
||||
tournaments = result.get('tournaments', []) |
||||
self.stdout.write(self.style.SUCCESS(f"✅ SUCCESS: {len(tournaments)} tournaments found")) |
||||
|
||||
if tournaments: |
||||
self.stdout.write("\n📝 Sample tournament:") |
||||
sample = tournaments[0] |
||||
self.stdout.write(f" ID: {sample.get('id')}") |
||||
self.stdout.write(f" Name: {sample.get('libelle')}") |
||||
self.stdout.write(f" Date: {sample.get('dateDebut', {}).get('date', 'N/A')}") |
||||
self.stdout.write(f" Judge: {sample.get('jugeArbitre', {}).get('nom', 'N/A')}") |
||||
|
||||
self.stdout.write(f"\n📊 Summary:") |
||||
self.stdout.write(f" Total tournaments: {len(tournaments)}") |
||||
self.stdout.write(f" Pages scraped: {result.get('pages_scraped', 1)}") |
||||
|
||||
else: |
||||
self.stdout.write(self.style.ERROR("❌ FAILED: No tournaments found")) |
||||
|
||||
except Exception as e: |
||||
self.stdout.write(self.style.ERROR(f"❌ ERROR: {e}")) |
||||
import traceback |
||||
if verbose: |
||||
self.stdout.write(traceback.format_exc()) |
||||
Loading…
Reference in new issue