{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0os-Y4UmubfB"
      },
      "source": [
        "# **DESCARGAS**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GWpLrhC3ZLc-"
      },
      "outputs": [],
      "source": [
        "!pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121\n",
        "!pip3 install unsloth sentence-transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0QQRu5-ARNuX"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qv6lpl6FuYx4"
      },
      "source": [
        "# **LORAS**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rao3cWeVVGsJ"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset\n",
        "from unsloth import FastLanguageModel\n",
        "from transformers import TextStreamer\n",
        "from difflib import SequenceMatcher\n",
        "import random\n",
        "\n",
        "# Carga del modelo\n",
        "model_path = \"/content/drive/MyDrive/Qwen2.5-0.5B-1ep-Server/loras/Qwen2.5-0.5B\"\n",
        "\n",
        "model, tokenizer = FastLanguageModel.from_pretrained(\n",
        "    model_name=model_path,\n",
        "    max_seq_length=4096,\n",
        "    dtype=None,\n",
        "    load_in_4bit=False,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DHVYCQnGeqRk"
      },
      "outputs": [],
      "source": [
        "# Función principal: diccionario/traductor experto en llionés\n",
        "def lliones_expert(message: str) -> str:\n",
        "    try:\n",
        "        messages = [\n",
        "            {\"role\": \"system\", \"content\": \"Eres un Diccionario/Traductor experto en Leonés\"},\n",
        "            {\"role\": \"user\", \"content\": message},\n",
        "        ]\n",
        "\n",
        "        prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
        "        inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n",
        "\n",
        "        outputs = model.generate(**inputs, max_new_tokens=256)\n",
        "        decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
        "\n",
        "        # Procesar respuesta (opcional)\n",
        "        if \"assistant\" in decoded:\n",
        "            response = decoded.split(\"assistant\")[-1].strip()\n",
        "        else:\n",
        "            response = decoded.strip()\n",
        "\n",
        "        return response\n",
        "\n",
        "    except Exception as e:\n",
        "        return f\"❌ Error: {str(e)}\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qyeui-BdKuPs"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "import random\n",
        "from difflib import SequenceMatcher\n",
        "from datasets import load_dataset\n",
        "from sentence_transformers import SentenceTransformer, util\n",
        "\n",
        "# Cargar modelo de embeddings semánticos\n",
        "print(\"🔄 Cargando modelo de embeddings...\")\n",
        "model_emb = SentenceTransformer(\"sentence-transformers/all-MiniLM-L6-v2\")\n",
        "\n",
        "# Cargar dataset desde Hugging Face\n",
        "dataset = load_dataset(\"unileon-robotics/lliones-dict-tr\", split=\"train\")\n",
        "dataset = dataset.shuffle(seed=42).select(range(2000))  # Muestra reproducible\n",
        "\n",
        "# Función de similitud difusa\n",
        "def fuzzy_similarity(a: str, b: str) -> float:\n",
        "    return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio() * 100\n",
        "\n",
        "# Función de similitud semántica\n",
        "def semantic_similarity(a: str, b: str) -> float:\n",
        "    emb1 = model_emb.encode(a, convert_to_tensor=True)\n",
        "    emb2 = model_emb.encode(b, convert_to_tensor=True)\n",
        "    score = util.cos_sim(emb1, emb2).item()\n",
        "    return score * 100"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QcP1lbOUBeEr"
      },
      "outputs": [],
      "source": [
        "results = []\n",
        "\n",
        "# Evaluación\n",
        "for i, sample in enumerate(dataset):\n",
        "    entrada = sample[\"input\"]\n",
        "    esperado = sample[\"output\"]\n",
        "\n",
        "    predicho = lliones_expert(entrada)\n",
        "\n",
        "    fuzzy = fuzzy_similarity(predicho, esperado)\n",
        "    semantic = semantic_similarity(predicho, esperado)\n",
        "    similitud = max(fuzzy, semantic)  # Combinado: usamos el más alto\n",
        "\n",
        "    print(f\"Ejemplo {i+1}\")\n",
        "\n",
        "    if random.random() > 0.01:\n",
        "        print(f\"\\n📌 Ejemplo {i+1}\")\n",
        "        print(f\"🔹 Entrada     : {entrada}\")\n",
        "        print(f\"✅ Esperado    : {esperado}\")\n",
        "        print(f\"🤖 Predicho    : {predicho}\")\n",
        "        print(f\"📊 Fuzzy       : {fuzzy:.2f}%\")\n",
        "        print(f\"💡 Semántica   : {semantic:.2f}%\")\n",
        "        print(f\"🏁 Final (max) : {similitud:.2f}%\")\n",
        "\n",
        "    results.append(similitud)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dsaHIC7GC3N1"
      },
      "outputs": [],
      "source": [
        "# Estadísticas finales\n",
        "media_similitud = sum(results) / len(results)\n",
        "print(f\"\\n📈 Similitud media en {len(results)} ejemplos: {media_similitud:.2f}%\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8p5A9AZDQXBE"
      },
      "outputs": [],
      "source": [
        "# Contador por rangos\n",
        "from collections import Counter\n",
        "\n",
        "# Definir rangos (de 5 en 5)\n",
        "rangos = [(i, i + 5) for i in range(0, 100, 5)]  # [(0,5), (5,10), ..., (95,100)]\n",
        "contador_rangos = Counter()\n",
        "\n",
        "for sim in results:\n",
        "    for (rango_min, rango_max) in rangos:\n",
        "        if rango_min <= sim < rango_max:\n",
        "            etiqueta = f\"{rango_min:02d}-{rango_max:02d}\"\n",
        "            contador_rangos[etiqueta] += 1\n",
        "            break\n",
        "    else:\n",
        "        if sim >= 100:\n",
        "            contador_rangos[\"100\"] += 1  # Para valores exactamente 100\n",
        "\n",
        "# Mostrar resultados ordenados\n",
        "print(\"\\n📊 Distribución de similitudes por rango:\")\n",
        "for rango in sorted(contador_rangos.keys(), reverse=True):\n",
        "    cantidad = contador_rangos[rango]\n",
        "    porcentaje = (cantidad / len(results)) * 100\n",
        "    print(f\"  {rango}%: {cantidad} ejemplos ({porcentaje:.1f}%)\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UbGgvBmT4puG"
      },
      "source": [
        "## Codigo para probar el Modelo Loras"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MPG5o5EM4lP4"
      },
      "outputs": [],
      "source": [
        "# # 🔁 Bucle interactivo de chat para modelo LoRA\n",
        "# print(\"🦙 Chat con tu modelo LoRA. Escribe 'salir' para terminar.\\n\")\n",
        "\n",
        "# while True:\n",
        "#     user_input = input(\"🧑 Tú: \")\n",
        "#     if user_input.lower() in [\"salir\", \"exit\", \"quit\"]:\n",
        "#         print(\"👋 Hasta pronto.\")\n",
        "#         break\n",
        "\n",
        "#     respuesta = lliones_expert(user_input)\n",
        "#     print(f\"🤖 LLM: {respuesta}\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yQhHyCrpuTpd"
      },
      "source": [
        "# **GGUF**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "3kp7hBUuCqqu"
      },
      "outputs": [],
      "source": [
        "from llama_cpp import Llama\n",
        "from datasets import load_dataset\n",
        "from difflib import SequenceMatcher\n",
        "from collections import Counter\n",
        "from sentence_transformers import SentenceTransformer, util\n",
        "import random\n",
        "import time\n",
        "import json\n",
        "import csv\n",
        "from pathlib import Path\n",
        "\n",
        "# ---------- Configuración ----------\n",
        "# Tamaño de muestra del dataset (ajústalo si quieres)\n",
        "SAMPLE_SIZE = 3000\n",
        "\n",
        "# Rutas de los modelos (en el orden que pasaste)\n",
        "MODEL_PATHS = [\n",
        "\n",
        "    ### RUTAS A GGUF ###\n",
        "\n",
        "]\n",
        "\n",
        "# Parámetros de inferencia llama.cpp (ajústalos si lo necesitas)\n",
        "LLAMA_KWARGS = dict(\n",
        "    n_ctx=2048,\n",
        "    n_threads=4,\n",
        "    n_gpu_layers=-1,\n",
        "    verbose=True,\n",
        ")\n",
        "\n",
        "# Semilla para reproducibilidad del muestreo del dataset\n",
        "RANDOM_SEED = 42\n",
        "\n",
        "# ---------- Guardar resumen comparativo en Drive ----------\n",
        "OUT_DIR = Path(\"/content/drive/MyDrive/lliones_eval/F16/Qwen2.5-3B-3ep\")\n",
        "OUT_DIR.mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "csv_path = OUT_DIR / \"lliones_eval_summary.csv\"\n",
        "json_path = OUT_DIR / \"lliones_eval_summary.json\"\n",
        "\n",
        "# ---------- Cargar modelo semántico (una sola vez) ----------\n",
        "semantic_model = SentenceTransformer(\"sentence-transformers/all-MiniLM-L6-v2\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O3QLCpDr0Ggj"
      },
      "outputs": [],
      "source": [
        "# ---------- Cargar dataset y seleccionar muestra reproducible ----------\n",
        "dataset = load_dataset(\"unileon-robotics/lliones-dict-tr\", split=\"train\")\n",
        "dataset = dataset.shuffle(seed=RANDOM_SEED).select(range(min(SAMPLE_SIZE, len(dataset))))\n",
        "\n",
        "def fuzzy_similarity(a: str, b: str) -> float:\n",
        "    return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio() * 100\n",
        "\n",
        "def semantic_similarity(a: str, b: str) -> float:\n",
        "    embeddings = semantic_model.encode([a, b], convert_to_tensor=True)\n",
        "    sim = util.cos_sim(embeddings[0], embeddings[1]).item()\n",
        "    return sim * 100  # porcentaje\n",
        "\n",
        "def stats_distribution(lista):\n",
        "    \"\"\"Devuelve un dict con la distribución en rangos 00-05, 05-10, ..., 95-100/100.\"\"\"\n",
        "    rangos = [(i, i + 5) for i in range(0, 100, 5)]\n",
        "    contador = Counter()\n",
        "    for sim in lista:\n",
        "        placed = False\n",
        "        for (rmin, rmax) in rangos:\n",
        "            if rmin <= sim < rmax:\n",
        "                etiqueta = f\"{rmin:02d}-{rmax:02d}\"\n",
        "                contador[etiqueta] += 1\n",
        "                placed = True\n",
        "                break\n",
        "        if not placed:\n",
        "            if sim >= 100:\n",
        "                contador[\"100\"] += 1\n",
        "    return dict(contador)\n",
        "\n",
        "def print_stats(lista):\n",
        "    media = sum(lista) / len(lista)\n",
        "    print(f\"\\n📈 Similitud media (máx fuzzy/semántica): {media:.2f}%\")\n",
        "    dist = stats_distribution(lista)\n",
        "    print(\"\\n📊 Distribución de similitudes:\")\n",
        "    for rango in sorted(dist.keys(), reverse=True):\n",
        "        cantidad = dist[rango]\n",
        "        porcentaje = (cantidad / len(lista)) * 100\n",
        "        print(f\"  {rango}%: {cantidad} ejemplos ({porcentaje:.1f}%)\")\n",
        "    return media, dist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b_nD30bd1l_-"
      },
      "outputs": [],
      "source": [
        "def build_llm(model_path: str) -> Llama:\n",
        "    return Llama(model_path=model_path, **LLAMA_KWARGS)\n",
        "\n",
        "def lliones_expert_gguf(message: str, llm: Llama) -> str:\n",
        "    # Prompt tipo chat Qwen\n",
        "    prompt = (\n",
        "        \"<|im_start|>system\\n\"\n",
        "        \"Eres un Diccionario/Traductor experto en leonés.<|im_end|>\\n\"\n",
        "        \"<|im_start|>user\\n\"\n",
        "        f\"{message}<|im_end|>\\n\"\n",
        "        \"<|im_start|>assistant\\n\"\n",
        "    )\n",
        "    output = llm(prompt, max_tokens=200)\n",
        "    respuesta = output[\"choices\"][0][\"text\"].strip()\n",
        "    return respuesta\n",
        "\n",
        "def evaluate_model(model_path: str, dataset, sample_print_prob: float = 0.01):\n",
        "    print(f\"\\n==============================\")\n",
        "    print(f\"🚀 Evaluando modelo: {model_path}\")\n",
        "    print(f\"==============================\")\n",
        "\n",
        "    llm = build_llm(model_path)\n",
        "    final_scores = []\n",
        "    fuzzy_scores = []\n",
        "    semantic_scores = []\n",
        "    printed = 0\n",
        "\n",
        "    t0 = time.time()\n",
        "    for i, sample in enumerate(dataset):\n",
        "        entrada = sample[\"input\"]\n",
        "        esperado = sample[\"output\"]\n",
        "        predicho = lliones_expert_gguf(entrada, llm=llm)\n",
        "\n",
        "        fuzzy_sim = fuzzy_similarity(predicho, esperado)\n",
        "        semantic_sim = semantic_similarity(predicho, esperado)\n",
        "        final_sim = max(fuzzy_sim, semantic_sim)\n",
        "        print(f\"\\nEjemplo {i+1}\\n\")\n",
        "        if random.random() < sample_print_prob:\n",
        "            printed += 1\n",
        "            print(f\"🔹 Entrada   : {entrada}\")\n",
        "            print(f\"✅ Esperado  : {esperado}\")\n",
        "            print(f\"🤖 Predicho  : {predicho}\")\n",
        "            print(f\"📊 Fuzzy     : {fuzzy_sim:.2f}%\")\n",
        "            print(f\"🧠 Semántico : {semantic_sim:.2f}%\")\n",
        "            print(f\"🏁 Final     : {final_sim:.2f}%\")\n",
        "\n",
        "        final_scores.append(final_sim)\n",
        "        fuzzy_scores.append(fuzzy_sim)\n",
        "        semantic_scores.append(semantic_sim)\n",
        "\n",
        "    elapsed = time.time() - t0\n",
        "\n",
        "    # Medias\n",
        "    mean_final = sum(final_scores) / len(final_scores)\n",
        "    mean_fuzzy = sum(fuzzy_scores) / len(fuzzy_scores)\n",
        "    mean_semantic = sum(semantic_scores) / len(semantic_scores)\n",
        "\n",
        "    # Estadísticas de la métrica final (como hasta ahora)\n",
        "    media_impresa, dist = print_stats(final_scores)\n",
        "    # Asegura que lo impreso coincide con mean_final (por claridad)\n",
        "    print(f\"\\n🔁 Comprobación medias -> Final: {mean_final:.2f}% | Fuzzy: {mean_fuzzy:.2f}% | Semántica: {mean_semantic:.2f}%\")\n",
        "    print(f\"⏱️ Tiempo total: {elapsed:.1f} s  |  {len(dataset)} ejemplos  |  Muestras impresas: {printed}\")\n",
        "\n",
        "    return {\n",
        "        \"model_path\": model_path,\n",
        "        \"num_examples\": len(dataset),\n",
        "        \"mean_similarity\": mean_final,     # métrica final (max por ejemplo) — se mantiene igual\n",
        "        \"mean_fuzzy\": mean_fuzzy,          # nueva media fuzzy\n",
        "        \"mean_semantic\": mean_semantic,    # nueva media semántica\n",
        "        \"distribution\": dist,              # distribución de la métrica final\n",
        "        \"elapsed_seconds\": elapsed,\n",
        "    }\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LhQn7b7EbxYD"
      },
      "outputs": [],
      "source": [
        "all_summaries = []\n",
        "for path in MODEL_PATHS:\n",
        "    summary = evaluate_model(path, dataset)\n",
        "    all_summaries.append(summary)\n",
        "\n",
        "# CSV: una fila por modelo (añadimos mean_fuzzy y mean_semantic)\n",
        "import csv, json\n",
        "with open(csv_path, \"w\", newline=\"\", encoding=\"utf-8\") as f:\n",
        "    writer = csv.writer(f)\n",
        "    writer.writerow([\n",
        "        \"model_path\",\n",
        "        \"num_examples\",\n",
        "        \"mean_similarity\",   # final (max por ejemplo)\n",
        "        \"mean_fuzzy\",\n",
        "        \"mean_semantic\",\n",
        "        \"elapsed_seconds\",\n",
        "        \"distribution_json\"\n",
        "    ])\n",
        "    for s in all_summaries:\n",
        "        writer.writerow([\n",
        "            s[\"model_path\"],\n",
        "            s[\"num_examples\"],\n",
        "            f\"{s['mean_similarity']:.4f}\",\n",
        "            f\"{s['mean_fuzzy']:.4f}\",\n",
        "            f\"{s['mean_semantic']:.4f}\",\n",
        "            f\"{s['elapsed_seconds']:.2f}\",\n",
        "            json.dumps(s[\"distribution\"], ensure_ascii=False, separators=(\",\", \":\")),\n",
        "        ])\n",
        "\n",
        "# JSON: objeto con toda la info (incluye las nuevas medias)\n",
        "with open(json_path, \"w\", encoding=\"utf-8\") as f:\n",
        "    json.dump(all_summaries, f, ensure_ascii=False, indent=2)\n",
        "\n",
        "print(\"\\n📁 Resultados guardados en tu Drive:\")\n",
        "print(f\" - CSV : {csv_path}\")\n",
        "print(f\" - JSON: {json_path}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rZKZdCYW3rw3"
      },
      "source": [
        "## Codigo para probar el Modelo GGUF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "tF8o87f-CrR4"
      },
      "outputs": [],
      "source": [
        "# # Bucle interactivo de chat\n",
        "# print(\"🦙 Chat con tu modelo GGUF. Escribe 'salir' para terminar.\\n\")\n",
        "\n",
        "# while True:\n",
        "#     user_input = input(\"🧑 Tú: \")\n",
        "#     if user_input.lower() in [\"salir\", \"exit\", \"quit\"]:\n",
        "#         print(\"👋 Hasta pronto.\")\n",
        "#         break\n",
        "\n",
        "#     prompt = (\n",
        "#         \"<|im_start|>system\\n\"\n",
        "#         \"Eres un Diccionario/Traductor experto en leonés.<|im_end|>\\n\"\n",
        "#         \"<|im_start|>user\\n\"\n",
        "#         f\"{user_input}<|im_end|>\\n\"\n",
        "#         \"<|im_start|>assistant\\n\"\n",
        "#     )\n",
        "\n",
        "#     output = llm(prompt, max_tokens=200)\n",
        "#     respuesta = output[\"choices\"][0][\"text\"].strip()\n",
        "#     print(f\"🤖 LLM: {respuesta}\\n\")\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}