#!/usr/bin/env python3
"""
Ghost Agency Radar v3 - Multi-source Lead Generation con Email Reali
Fonti: Seed CSV + Overpass API (OSM) + Facebook/Google Maps scraping + Hunter.io + Pattern MX
Output: CSV con email verificate + call_list per chi non ha email
"""

import os
import csv
import time
import socket
import random
import requests
import json
import re
from pathlib import Path
from datetime import datetime
from urllib.parse import quote_plus

# Config
SEED_CSV = Path("/home/ubuntu/GhostAgency/leads_seed_parma.csv")
OUTPUT_DIR = Path("/home/ubuntu/GhostAgency")
HUNTER_API_KEY = os.getenv("HUNTER_API_KEY", "")  # Opzionale
GOOGLE_MAPS_API_KEY = os.getenv("GOOGLE_MAPS_API_KEY", "")  # Opzionale
FACEBOOK_TOKEN = os.getenv("FACEBOOK_TOKEN", "")  # Opzionale

# Rate limits
DNS_DELAY = 0.3
HUNTER_DELAY = 1.0
SCRAPE_DELAY = 2.0

# User agents for scraping
USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
]

def get_headers():
    return {"User-Agent": random.choice(USER_AGENTS)}

# ============================================================
# DNS / MX CHECKS
# ============================================================
def check_dns(domain):
    """Verifica se dominio esiste (A record)."""
    try:
        socket.gethostbyname(domain)
        return True
    except socket.gaierror:
        return False

def check_mx(domain):
    """Verifica se dominio ha record MX (accetta email)."""
    try:
        import dns.resolver
        answers = dns.resolver.resolve(domain, 'MX')
        return len(answers) > 0
    except Exception:
        return False

# ============================================================
# EMAIL PATTERN GENERATION + VERIFICATION
# ============================================================
def generate_email_patterns(nome, city="parma"):
    """Genera pattern email comuni per un nome attività."""
    clean = "".join(c for c in nome.lower() if c.isalnum())
    city_clean = city.lower()
    
    patterns = []
    
    # Pattern principali
    for prefix in ["info", "contatto", "prenotazioni", "admin", "mail", "segreteria", "ufficio"]:
        for domain_ext in [".it", ".com"]:
            for domain_base in [clean, f"{clean}{city_clean}", f"{clean}-{city_clean}"]:
                patterns.append(f"{prefix}@{domain_base}{domain_ext}")
    
    # Pattern specifici per categoria (verranno filtrati dopo)
    return list(dict.fromkeys(patterns))  # deduplica

def verify_email_mx(email):
    """Verifica email via MX record del dominio."""
    domain = email.split("@")[1]
    return check_mx(domain)

# ============================================================
# HUNTER.IO API (25 ricerche gratis/mese)
# ============================================================
def hunter_find_email(domain, nome):
    """Trova email via Hunter.io Domain Search."""
    if not HUNTER_API_KEY:
        return None
    
    try:
        url = "https://api.hunter.io/v2/domain-search"
        params = {
            "domain": domain,
            "api_key": HUNTER_API_KEY,
            "limit": 10,
            "type": "generic"
        }
        response = requests.get(url, params=params, timeout=10)
        if response.status_code == 200:
            data = response.json()
            emails = data.get("data", {}).get("emails", [])
            # Filtra per confidence > 80%
            for e in emails:
                if e.get("confidence", 0) >= 80:
                    return e.get("value")
    except Exception as e:
        print(f"    [Hunter] Errore: {e}")
    return None

# ============================================================
# GOOGLE MAPS PLACES API - Trova sito web + email
# ============================================================
def gmaps_find_place(nome, city="Parma"):
    """Trova place_id e dettagli via Google Maps Places API."""
    if not GOOGLE_MAPS_API_KEY:
        return None
    
    try:
        # 1. Find Place
        url = "https://maps.googleapis.com/maps/api/place/findplacefromtext/json"
        params = {
            "input": f"{nome}, {city}",
            "inputtype": "textquery",
            "fields": "place_id,name,formatted_address,website,formatted_phone_number,email",
            "key": GOOGLE_MAPS_API_KEY
        }
        response = requests.get(url, params=params, timeout=10)
        if response.status_code == 200:
            data = response.json()
            candidates = data.get("candidates", [])
            if candidates:
                place_id = candidates[0].get("place_id")
                # 2. Place Details per website + email
                return gmaps_place_details(place_id)
    except Exception as e:
        print(f"    [GMaps] Errore: {e}")
    return None

def gmaps_place_details(place_id):
    """Ottieni dettagli completi incluso website."""
    if not GOOGLE_MAPS_API_KEY:
        return None
    
    try:
        url = "https://maps.googleapis.com/maps/api/place/details/json"
        params = {
            "place_id": place_id,
            "fields": "name,website,formatted_phone_number,email,url",
            "key": GOOGLE_MAPS_API_KEY
        }
        response = requests.get(url, params=params, timeout=10)
        if response.status_code == 200:
            return response.json().get("result", {})
    except Exception as e:
        print(f"    [GMaps Details] Errore: {e}")
    return None

# ============================================================
# FACEBOOK GRAPH API - Scraping pagine pubbliche
# ============================================================
def facebook_find_page(nome, city="Parma"):
    """Cerca pagina Facebook pubblica."""
    if not FACEBOOK_TOKEN:
        return None
    
    try:
        url = "https://graph.facebook.com/v18.0/search"
        params = {
            "type": "page",
            "q": f"{nome} {city}",
            "fields": "id,name,website,email,phone,link",
            "access_token": FACEBOOK_TOKEN,
            "limit": 5
        }
        response = requests.get(url, params=params, timeout=10)
        if response.status_code == 200:
            data = response.json()
            pages = data.get("data", [])
            if pages:
                return pages[0]  # Prima corrispondenza
    except Exception as e:
        print(f"    [FB] Errore: {e}")
    return None

def facebook_get_page_details(page_id):
    """Ottieni dettagli pagina (website, email)."""
    if not FACEBOOK_TOKEN:
        return None
    
    try:
        url = f"https://graph.facebook.com/v18.0/{page_id}"
        params = {
            "fields": "website,email,phone,link,about",
            "access_token": FACEBOOK_TOKEN
        }
        response = requests.get(url, params=params, timeout=10)
        if response.status_code == 200:
            return response.json()
    except Exception as e:
        print(f"    [FB Details] Errore: {e}")
    return None

# ============================================================
# WEB SCRAPING - Estrai email da sito web
# ============================================================
def scrape_website_emails(url):
    """Estrai email da pagina web."""
    try:
        resp = requests.get(url, headers=get_headers(), timeout=10, allow_redirects=True)
        if resp.status_code == 200:
            # Regex per email
            emails = re.findall(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', resp.text)
            # Filtra email valide (no image, no css, etc.)
            valid = []
            for e in emails:
                e = e.lower()
                if not any(skip in e for skip in [".png", ".jpg", ".jpeg", ".gif", ".css", ".js", ".woff", ".svg", "example.com", "test.com", "localhost"]):
                    valid.append(e)
            return list(dict.fromkeys(valid))
    except Exception as e:
        print(f"    [Scrape] Errore {url}: {e}")
    return []

def find_contact_page(base_url):
    """Trova pagina contatti."""
    contact_paths = ["/contatti", "/contatto", "/contacts", "/contact", "/chi-siamo", "/about", "/prenota", "/prenotazioni"]
    for path in contact_paths:
        test_url = base_url.rstrip("/") + path
        try:
            resp = requests.head(test_url, headers=get_headers(), timeout=5, allow_redirects=True)
            if resp.status_code == 200:
                return test_url
        except:
            continue
    return base_url

# ============================================================
# OVERPASS API (OSM) - Trova attività senza sito
# ============================================================
def overpass_query(bbox, categories):
    """Query Overpass API per POI senza website."""
    overpass_url = "https://overpass-api.de/api/interpreter"
    
    # Costruisci query per categorie multiple
    queries = []
    for cat in categories:
        queries.append(f'node["{cat}"]({bbox});')
        queries.append(f'way["{cat}"]({bbox});')
    
    query = f"""
    [out:json][timeout:60];
    (
        {' '.join(queries)}
    );
    out center tags;
    """
    
    try:
        resp = requests.post(overpass_url, data=query, timeout=60)
        if resp.status_code == 200:
            return resp.json().get("elements", [])
    except Exception as e:
        print(f"    [Overpass] Errore: {e}")
    return []

# ============================================================
# SEED CSV MANAGEMENT
# ============================================================
def create_seed_if_missing():
    if SEED_CSV.exists():
        return
    
    seed_data = [
        # Ristoranti/Pizzerie
        {"nome": "Pizzeria Da Mario", "telefono": "0521 123456", "indirizzo": "Via Garibaldi 15, Parma", "categoria": "Ristoranti"},
        {"nome": "Trattoria del Tribunale", "telefono": "0521 234567", "indirizzo": "Via del Tribunale 8, Parma", "categoria": "Ristoranti"},
        {"nome": "Ristorante La Greppia", "telefono": "0521 345678", "indirizzo": "Strada della Greppia 12, Parma", "categoria": "Ristoranti"},
        {"nome": "Pizzeria Napoletana", "telefono": "0521 456789", "indirizzo": "Via Mazzini 22, Parma", "categoria": "Ristoranti"},
        {"nome": "Osteria dei Medici", "telefono": "0521 567890", "indirizzo": "Via dei Medici 5, Parma", "categoria": "Ristoranti"},
        
        # Bar/Caffè
        {"nome": "Caffè Centrale", "telefono": "0521 678901", "indirizzo": "Piazza Garibaldi 1, Parma", "categoria": "Bar/Caffè"},
        {"nome": "Bar Sport", "telefono": "0521 789012", "indirizzo": "Viale Mentana 45, Parma", "categoria": "Bar/Caffè"},
        {"nome": "Pasticceria Torino", "telefono": "0521 890123", "indirizzo": "Via Farini 33, Parma", "categoria": "Bar/Caffè"},
        
        # Dentisti
        {"nome": "Studio Dentistico Parma Centro", "telefono": "0521 901234", "indirizzo": "Via della Repubblica 10, Parma", "categoria": "Dentisti"},
        {"nome": "Dott. Rossi Odontoiatra", "telefono": "0521 012345", "indirizzo": "Strada Langhirano 55, Parma", "categoria": "Dentisti"},
        
        # Idraulici
        {"nome": "Idraulica Parma Service", "telefono": "0521 112233", "indirizzo": "Via Sidoli 20, Parma", "categoria": "Idraulici"},
        {"nome": "Pronto Intervento Idraulico", "telefono": "338 1234567", "indirizzo": "Parma", "categoria": "Idraulici"},
        
        # Elettricisti
        {"nome": "Elettronica Moderna", "telefono": "0521 223344", "indirizzo": "Via Trento 8, Parma", "categoria": "Elettricisti"},
        
        # Autofficine
        {"nome": "Autofficina Meccanica Parma", "telefono": "0521 334455", "indirizzo": "Via Mantova 100, Parma", "categoria": "Autofficine"},
        
        # Parrucchieri
        {"nome": "Salone Bellezza Parma", "telefono": "0521 445566", "indirizzo": "Via Cavour 18, Parma", "categoria": "Parrucchieri"},
        
        # Centri Estetici
        {"nome": "Estetica & Benessere", "telefono": "0521 556677", "indirizzo": "Viale Piacenza 25, Parma", "categoria": "Centri Estetici"},
        
        # Farmacie
        {"nome": "Farmacia Comunale 1", "telefono": "0521 667788", "indirizzo": "Piazza Ghiaia 3, Parma", "categoria": "Farmacie"},
        
        # Panifici
        {"nome": "Panificio Artigianale Parma", "telefono": "0521 778899", "indirizzo": "Via Bixio 12, Parma", "categoria": "Panifici"},
    ]
    
    fieldnames = ["nome", "telefono", "indirizzo", "categoria"]
    with open(SEED_CSV, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(seed_data)
    
    print(f"[+] Creato seed CSV con {len(seed_data)} attività: {SEED_CSV}")

# ============================================================
# MAIN PIPELINE: FIND EMAIL FOR LEAD
# ============================================================
def find_email_for_lead(nome, telefono, indirizzo, categoria, city="Parma"):
    """
    Pipeline completo per trovare email:
    1. Hunter.io (se hai dominio)
    2. Google Maps (trova sito web)
    3. Facebook (pagina pubblica)
    4. Web scraping (se ha sito)
    5. Pattern MX verification
    """
    print(f"    [*] Ricerca email per: {nome}")
    
    # 1. Prova Hunter.io se riesci a indovinare dominio
    clean = "".join(c for c in nome.lower() if c.isalnum())
    guessed_domain = f"{clean}.it"
    if check_dns(guessed_domain) and HUNTER_API_KEY:
        email = hunter_find_email(guessed_domain, nome)
        if email and verify_email_mx(email):
            print(f"    [+] Hunter.io: {email}")
            return email, "hunter"
    
    # 2. Google Maps Places API
    if GOOGLE_MAPS_API_KEY:
        place = gmaps_find_place(nome, city)
        if place:
            website = place.get("website")
            if website:
                print(f"    [*] Trovato sito via Google Maps: {website}")
                # Scrapa sito per email
                emails = scrape_website_emails(website)
                contact_url = find_contact_page(website)
                if contact_url != website:
                    emails += scrape_website_emails(contact_url)
                if emails:
                    # Verifica MX per ogni email
                    for e in emails:
                        if verify_email_mx(e):
                            print(f"    [+] Scraping sito: {e}")
                            return e, "gmaps_scrape"
            # Controlla se Google Maps ha email diretta (raro)
            if place.get("email"):
                email = place["email"]
                if verify_email_mx(email):
                    print(f"    [+] Google Maps diretto: {email}")
                    return email, "gmaps_direct"
    
    # 3. Facebook Graph API
    if FACEBOOK_TOKEN:
        fb_page = facebook_find_page(nome, city)
        if fb_page:
            details = facebook_get_page_details(fb_page.get("id"))
            if details:
                if details.get("email"):
                    email = details["email"]
                    if verify_email_mx(email):
                        print(f"    [+] Facebook: {email}")
                        return email, "facebook"
                if details.get("website"):
                    emails = scrape_website_emails(details["website"])
                    for e in emails:
                        if verify_email_mx(e):
                            print(f"    [+] Facebook sito: {e}")
                            return e, "facebook_scrape"
    
    # 4. Pattern MX verification (ultima risorsa)
    print(f"    [*] Tentativo pattern MX...")
    for pattern in generate_email_patterns(nome, city)[:20]:  # Limita a 20
        if verify_email_mx(pattern):
            print(f"    [+] Pattern MX: {pattern}")
            return pattern, "pattern_mx"
        time.sleep(0.1)
    
    return None, "none"

# ============================================================
# RADAR MAIN
# ============================================================
def run_radar():
    print(f"\n{'='*60}")
    print(f" GHOST AGENCY RADAR v3 - Multi-source Email Discovery")
    print(f" Target: Parma | Fonti: Hunter, Google Maps, FB, Scraping, MX")
    print(f"{'='*60}\n")
    
    create_seed_if_missing()
    
    # Leggi seed
    with open(SEED_CSV, newline="", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        leads = list(reader)
    
    print(f"[*] Caricate {len(leads)} attività dal seed\n")
    
    # Carica CSV esistenti per non duplicare
    existing_emails = set()
    for csv_file in OUTPUT_DIR.glob("leads_parma_*_local.csv"):
        try:
            with open(csv_file, newline="", encoding="utf-8") as f:
                r = csv.DictReader(f)
                for row in r:
                    if row.get("Email"):
                        existing_emails.add(row["Email"].lower())
        except:
            pass
    
    # Carica sent_emails
    sent_emails = set()
    sent_file = OUTPUT_DIR / "sent_emails.json"
    if sent_file.exists():
        with open(sent_file) as f:
            sent_emails = set(json.load(f))
    
    all_existing = existing_emails | {e.lower() for e in sent_emails}
    
    valid_leads = []
    call_leads = []
    
    for i, row in enumerate(leads, 1):
        nome = row["nome"].strip()
        telefono = row["telefono"].strip()
        indirizzo = row["indirizzo"].strip()
        categoria = row["categoria"].strip()
        
        print(f"\n[{i}/{len(leads)}] {nome} ({categoria})")
        
        # Skip se già processato (controlla nome+telefono)
        lead_key = f"{nome}|{telefono}"
        
        email, source = find_email_for_lead(nome, telefono, indirizzo, categoria)
        
        if email:
            email_lower = email.lower()
            if email_lower in all_existing:
                print(f"    [=] Già in database: {email}")
                continue
            
            lead = {
                "Nome": nome,
                "Email": email,
                "Telefono": telefono,
                "Indirizzo": indirizzo,
                "Categoria": categoria,
                "Fonte": source,
                "Data": datetime.now().strftime("%Y-%m-%d")
            }
            valid_leads.append(lead)
            all_existing.add(email_lower)
            print(f"    [���] LEAD CON EMAIL: {nome} | {email} ({source})")
        else:
            # Nessuna email -> call list
            lead = {
                "Nome": nome,
                "Telefono": telefono,
                "Indirizzo": indirizzo,
                "Categoria": categoria,
                "Data": datetime.now().strftime("%Y-%m-%d")
            }
            call_leads.append(lead)
            print(f"    [��] CALL LIST: {nome} | {telefono}")
        
        # Rate limiting
        time.sleep(SCRAPE_DELAY)
    
    # Salvataggio CSV con email
    if valid_leads:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = f"leads_parma_{timestamp}_local.csv"
        filepath = OUTPUT_DIR / filename
        
        fieldnames = ["Nome", "Email", "Telefono", "Indirizzo", "Categoria", "Fonte", "Data"]
        with open(filepath, "w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=fieldnames)
            writer.writeheader()
            writer.writerows(valid_leads)
        
        print(f"\n[+] SALVATI {len(valid_leads)} LEAD CON EMAIL in {filename}")
    else:
        print(f"\n[-] Nessun lead con email trovato")
    
    # Salvataggio call list
    if call_leads:
        call_file = OUTPUT_DIR / "call_list.csv"
        file_exists = call_file.exists()
        
        fieldnames = ["Nome", "Telefono", "Indirizzo", "Categoria", "Data"]
        with open(call_file, "a", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=fieldnames)
            if not file_exists:
                writer.writeheader()
            writer.writerows(call_leads)
        
        print(f"[+] AGGIUNTI {len(call_leads)} A CALL LIST")
    
    return len(valid_leads), len(call_leads)

if __name__ == "__main__":
    run_radar()