import os
import re
import json
import glob
import pandas as pd
import spacy
from collections import Counter

CONVERSATION_DIR = "conversation/sacha-only"
OUTPUT_VOCAB_CSV = "french_spoken_vocabulary_list.csv"
OUTPUT_SUMMARY_CSV = "french_spoken_progress_summary.csv"

def parse_vtt(content):
    """Extracts plain text lines from a VTT file string."""
    lines = content.splitlines()
    text_lines = []
    for line in lines:
        line = line.strip()
        if not line or line.startswith("WEBVTT") or "-->" in line or line.isdigit():
            continue
        clean = re.sub(r'<[^>]+>', '', line)
        if clean:
            text_lines.append(clean)
    return " ".join(text_lines)

def parse_json(content):
    """Extracts text segments from a WhisperX JSON output."""
    data = json.loads(content)
    text_parts = []
    if "segments" in data:
        for seg in data["segments"]:
            if "text" in seg:
                text_parts.append(seg["text"].strip())
    elif "text" in data:
        text_parts.append(data["text"].strip())
    return " ".join(text_parts)

def read_transcript_content(filepath):
    """Reads file content based on extension."""
    ext = os.path.splitext(filepath)[1].lower()
    with open(filepath, "r", encoding="utf-8") as f:
        raw_content = f.read()

    if ext == ".txt":
        return raw_content.strip()
    elif ext == ".vtt":
        return parse_vtt(raw_content)
    elif ext == ".json":
        return parse_json(raw_content)
    return ""

def get_verb_tense_tag(token):
    """Analyzes a token to determine grammatical tense/mood matching journal logic."""
    if token.pos_ not in ["VERB", "AUX"]:
        return "N/A"

    morph = str(token.morph)

    if token.pos_ == "AUX" and token.dep_.startswith("aux") and "VerbForm=Part" in str(token.head.morph):
        return "N/A"

    is_marked_subordinate = any(child.dep_ == "mark" for child in token.children)
    if token.i > 0 and token.n_lefts > 0:
        left_children = list(token.children)
        if left_children and left_children[0].dep_ == "mark" and left_children[0].lower_ == 'que':
            is_marked_subordinate = True

    if is_marked_subordinate and token.lemma_ in ["pouvoir", "faire", "savoir", "vouloir", "aller", "être", "avoir"]:
        if "Mood=Ind" in morph or "Mood=Cnd" in morph:
            return "Subjonctif Corrigé (Règle)"

    children_deps = [child.dep_ for child in token.children]
    if token.pos_ == "VERB" and "VerbForm=Part" in morph and any(dep.startswith("aux") for dep in children_deps):
        return "Passé Composé"

    if token.lemma_ == "aller":
        for child in token.children:
            if child.dep_ == "xcomp" and "VerbForm=Inf" in str(child.morph):
                return "Futur Proche"

    if "Mood=Sub" in morph:
        if "Tense=Imp" in morph: return "Subjonctif Imparfait"
        return "Subjonctif"

    if "Mood=Cnd" in morph: return "Conditionnel"
    if "Tense=Fut" in morph: return "Futur Simple"
    if "Tense=Imp" in morph: return "Imparfait"
    if "Tense=Pres" in morph: return "Présent"
    if "VerbForm=Inf" in morph: return "Infinitif"
    if "Mood=Imp" in morph: return "Impératif"

    return "Autre/Participe"

def main():
    print("Loading French model (fr_dep_news_trf)...")
    try:
        nlp = spacy.load("fr_dep_news_trf")
    except OSError:
        print("Model not found. Please run: python -m spacy download fr_dep_news_trf")
        return

    # 1. Discover all candidate files
    all_files = glob.glob(os.path.join(CONVERSATION_DIR, "*"))

    # Structure: { date: { base_session_name: { extension: filepath } } }
    date_grouped_sessions = {}

    for filepath in all_files:
        filename = os.path.basename(filepath)
        base_name, ext = os.path.splitext(filename)
        ext = ext.lower()
        if ext in [".txt", ".vtt", ".json"]:
            date_tag = base_name[:10]  # Extract YYYY-MM-DD

            if date_tag not in date_grouped_sessions:
                date_grouped_sessions[date_tag] = {}

            if base_name not in date_grouped_sessions[date_tag]:
                date_grouped_sessions[date_tag][base_name] = {}

            date_grouped_sessions[date_tag][base_name][ext] = filepath

    if not date_grouped_sessions:
        print("No transcript files found in target folder.")
        return

    sorted_dates = sorted(date_grouped_sessions.keys())

    known_lemmas = set()
    summary_log = []
    vocab_log = []

    lemma_dates_map = {}
    cumulative_words = 0

    print(f"Found {len(sorted_dates)} unique recording dates across all sessions. Processing...")

    for i, date_tag in enumerate(sorted_dates):
        sessions_for_date = date_grouped_sessions[date_tag]

        # Combine text from all sessions on this date using file preference (TXT -> VTT -> JSON)
        date_text_parts = []
        selected_file_list = []

        for base_name in sorted(sessions_for_date.keys()):
            ext_dict = sessions_for_date[base_name]

            # Select format according to priority
            selected_ext = None
            for ext in [".txt", ".vtt", ".json"]:
                if ext in ext_dict:
                    selected_ext = ext
                    break

            selected_filepath = ext_dict[selected_ext]
            selected_file_list.append(os.path.basename(selected_filepath))

            text_content = read_transcript_content(selected_filepath)
            if text_content:
                date_text_parts.append(text_content)

        combined_text = " ".join(date_text_parts)
        print(f"Processing Date [{i+1}/{len(sorted_dates)}]: {date_tag} ({len(sessions_for_date)} session(s): {', '.join(selected_file_list)})")

        doc = nlp(combined_text)
        tokens = [token for token in doc if token.is_alpha]

        file_lemmas = set()
        lemma_counts = Counter()
        tense_counter = Counter()

        for token in tokens:
            if token.pos_ is None or token.pos_ == "PROPN":
                continue

            t_lemma = token.lemma_.lower()
            tense_tag = get_verb_tense_tag(token)

            if token.pos_ in ["VERB", "AUX"]:
                tense_counter[tense_tag] += 1

            lemma_counts[t_lemma] += 1
            file_lemmas.add(t_lemma)

            if t_lemma not in lemma_dates_map:
                lemma_dates_map[t_lemma] = set()
            lemma_dates_map[t_lemma].add(date_tag)

            vocab_log.append({
                "Date": date_tag,
                "Original Term": token.text.lower(),
                "Base Word (Lemma)": t_lemma,
                "POS": token.pos_,
                "Tense/Mood": tense_tag
            })

        # Output grouped date CSV (e.g., 2026-07-17.csv)
        date_csv_filename = f"{date_tag}.csv"
        date_csv_path = os.path.join(CONVERSATION_DIR, date_csv_filename)

        date_df = pd.DataFrame([
            {"Lemma": lem, "Frequency": count}
            for lem, count in lemma_counts.most_common()
        ])
        date_df.to_csv(date_csv_path, index=False, encoding="utf-8")

        # Summary calculations for the date
        new_lemmas = file_lemmas - known_lemmas
        new_lemmas_str = "" if i == 0 else ", ".join(sorted(new_lemmas))

        known_lemmas.update(new_lemmas)
        cumulative_words += len(tokens)

        summary_log.append({
            "Date": date_tag,
            "Total Word Count": len(tokens),
            "Cumulative Word Count": cumulative_words,
            "Unique Lemmas": len(file_lemmas),
            "New Lemmas": len(new_lemmas),
            "Cumulative Vocab": len(known_lemmas),
            "Count: Présent": tense_counter["Présent"],
            "Count: Passé Composé": tense_counter["Passé Composé"],
            "Count: Imparfait": tense_counter["Imparfait"],
            "Count: Futur Proche": tense_counter["Futur Proche"],
            "Count: Futur Simple": tense_counter["Futur Simple"],
            "Count: Subjonctif": tense_counter["Subjonctif"],
            "Count: Subjonctif Corrigé (Règle)": tense_counter["Subjonctif Corrigé (Règle)"],
            "Count: Conditionnel": tense_counter["Conditionnel"],
            "New Lemmas List": new_lemmas_str
        })

    # Export Vocabulary List
    vocab_df = pd.DataFrame(vocab_log)
    unique_dates_counts = {
        lemma: len(dates) for lemma, dates in lemma_dates_map.items()
    }
    vocab_df["Unique Dates Used"] = vocab_df["Base Word (Lemma)"].map(unique_dates_counts)
    vocab_df = vocab_df.sort_values(by="Unique Dates Used", ascending=True)
    vocab_df.to_csv(OUTPUT_VOCAB_CSV, index=False, encoding="utf-8")

    # Export Summary Log CSV
    pd.DataFrame(summary_log).to_csv(OUTPUT_SUMMARY_CSV, index=False, encoding="utf-8")

    print("\nProcessing completed successfully.")
    print(f"- Grouped per-date CSV files generated in {CONVERSATION_DIR}/ (e.g. YYYY-MM-DD.csv)")
    print(f"- Generated {OUTPUT_VOCAB_CSV} (sorted by unique dates used)")
    print(f"- Generated {OUTPUT_SUMMARY_CSV}")

if __name__ == "__main__":
    main()
