{
  "report": "002",
  "completed_on": "2026-10-13",
  "classification": "Exploratory internal adaptation; not scientifically validated; not peer reviewed",
  "verification": {
    "status": "verified locally",
    "method": "Retained archives and checkpoints reverified; predictions and paired uncertainty independently recomputed locally. Not external replication.",
    "raw_results_public": false,
    "weights_public": false,
    "training_archive_sha256": "a7fd387a692a8a19cd8b3f1f74078a8f4d921f66376284fef5a5750c4aaa2b44",
    "evaluation_archive_sha256": "4d6735e268eff0be3b11e4089bb5d7fe04c1e4921a85bf4ce96605f349097bb2",
    "training_manifest_sha256": "cfcc20faccbae7c656080530ee0d4402b33662de82f2449a6d96412c0e90bb98",
    "training_verification_sha256": "1f16929a5464a7456ef4ab00813da9f240c4add196a4726facd8ca208d8ff0b9",
    "evaluation_manifest_sha256": "acab972fd5551284ff17650ac8151becfa954b0dc80ba54d8236740721f674f9",
    "paired_comparison_sha256": "9d03ccd42619dae0c4aed26fedff61040809c4e7e65bc565bc748cb4d09e18b6",
    "base_samples_sha256": "e44bd4942d65e8ea03e519416d79f1f54dd53d2e7cf584d2addf5dc5ecce4512",
    "adapter_samples_sha256": "90e13a458ef3e774e6b69b966ea73064bfeb8a08b6e425ee1d4cc113def46a24",
    "plan_sha256": "a53f12b373d186b616803b6642840a27f6a7c57c9ec989e56190b65c2b17b4c3"
  },
  "model": {
    "eos_token_id": 151643,
    "repository": "Qwen/Qwen3-1.7B-Base",
    "revision": "ea980cb0a6c2ae4b936e82123acc929f1cec04c1",
    "tokenizer": "Qwen/Qwen3-1.7B-Base",
    "tokenizer_revision": "ea980cb0a6c2ae4b936e82123acc929f1cec04c1"
  },
  "aya": {
    "repository": "CohereLabs/aya_collection_language_split",
    "revision": "a3af2fde4b4cb5b2775830b11244a1a20b5f004f",
    "file": "filipino/train-00000-of-00001.parquet",
    "source_sha256": "8140b6bbdfca748fab7203bfe094574fabe0a0527b256dff0d07170357e4f1bc",
    "source_card_sha256": "453f2533dff0f575876bd5919e695db17f0b9c1d1b76b7d35e85ee0e9845bed1",
    "declared_license": "Apache-2.0; contributed third-party rights unresolved",
    "v1": {
      "original": 1241,
      "train": 437,
      "validation": 49,
      "quarantine": 742,
      "exclusions": 13
    },
    "v2": {
      "original": 1241,
      "raw_unique_pairs": 1233,
      "unique_raw_prompts": 939,
      "train": 951,
      "validation": 89,
      "quarantine": 184,
      "exclusions": 17,
      "usable_candidates": 1040
    },
    "selected_training": 363,
    "selected_groups": 363,
    "individual_agent_observations": 80,
    "screening_only": 283,
    "validation_candidates": 89,
    "provisional_validation_used": 66,
    "independent_human_review": false,
    "legal_clearance": false,
    "selected_training_sha256": "33a8edcaddee3c5b3fca12513b17ac906525b6d69745588b8a4c6cb848a071ac",
    "selected_validation_sha256": "a4b41287eee61c509f06ca5069338605233aeef90a684d19d1706fee89140705"
  },
  "training": {
    "settings": {
      "adapter_dtype": "float32",
      "bf16": false,
      "data_seed": 1234,
      "drop_last": false,
      "epochs": 1,
      "expected_optimizer_updates": 91,
      "gradient_accumulation_steps": 4,
      "gradient_checkpointing": true,
      "learning_rate": 0.0002,
      "lr_scheduler_type": "constant_with_warmup",
      "max_grad_norm": 1.0,
      "max_length": 512,
      "optimizer": "adamw_torch",
      "optimizer_state_dtype": "float32",
      "packing": false,
      "padding_labels": -100,
      "per_device_train_batch_size": 1,
      "prompt_labels": -100,
      "seed": 1234,
      "shuffle": true,
      "supervise_response_eos": true,
      "tf32": false,
      "truncation": false,
      "use_cache": false,
      "warmup_steps": 5,
      "weight_decay": 0.0
    },
    "quantization": {
      "bnb_4bit_compute_dtype": "float16",
      "bnb_4bit_quant_type": "nf4",
      "bnb_4bit_use_double_quant": true,
      "load_in_4bit": true
    },
    "lora": {
      "bias": "none",
      "lora_alpha": 32,
      "lora_dropout": 0.05,
      "r": 16,
      "target_modules": [
        "q_proj",
        "k_proj",
        "v_proj",
        "o_proj",
        "gate_proj",
        "up_proj",
        "down_proj"
      ],
      "task_type": "CAUSAL_LM"
    },
    "loss_scaling": {
      "backoff_factor": 0.5,
      "growth_factor": 2.0,
      "growth_interval": 2000,
      "init_scale": 1024.0
    },
    "metrics": {
      "examples_consumed": 363,
      "final_loss_scale": 1024.0,
      "final_training_loss": 1.3594228426615398,
      "initial_training_loss": 2.1746431291103363,
      "mean_training_loss": 1.6835284903440266,
      "optimizer_updates": 91,
      "peak_allocated_bytes": 3731268608,
      "peak_reserved_bytes": 4034920448,
      "trainable_parameters": 17432576,
      "training_seconds": 257.665785127
    },
    "started_at": "2026-10-12T17:41:36.995233+00:00",
    "finished_at": "2026-10-12T17:46:06.581895+00:00",
    "checkpoint_updates": [
      50,
      91
    ],
    "final_checkpoint": 91,
    "final_adapter_sha256": "f7605c629df52087d04da01fcf018368986570f64e3f864071eacb4a0ebdde2a",
    "checkpoint50_adapter_sha256": "7990dee87b8baebbe2cfb438dc54715651a6dda310c18fa47936ff016494211d",
    "proposal_sha256": "9ab753aa10954de2832187df4486b95507dd33e4446cb9b6402c9ad9588e6ac8",
    "reload": {
      "status": "passed",
      "fresh_process": true,
      "tensor_equality": true,
      "probe_finite": true,
      "probe_argmax_equal": true,
      "maximum_absolute_logit_error": 0.0,
      "tolerances": {
        "atol": 0.001,
        "rtol": 0.001
      }
    },
    "provisional_validation": {
      "samples": 66,
      "mean_example_cross_entropy_before": 2.0317009606144647,
      "mean_example_cross_entropy_after": 1.7707118744199926,
      "used_for_checkpoint_selection": false
    }
  },
  "protocol": {
    "task": "filbench|belebele_fil_mcf|0|0",
    "custom_entry_point": "community_tasks/filbench_evals.py",
    "filbench_revision": "7ad52795d7d218d4679f93a019b58dde4b749c5b",
    "few_shots": 0,
    "chat_template": false,
    "system_prompt": null,
    "batch_size": 1,
    "seed": 1234,
    "truncation": "abort",
    "uploads": false,
    "device": "cuda",
    "dtype": "float32",
    "threads": 4,
    "tf32": false,
    "quantization": false,
    "filtering": false
  },
  "benchmark": {
    "repository": "facebook/belebele",
    "revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
    "subset": "tgl_Latn",
    "split": "test",
    "samples": 900,
    "license": "CC-BY-SA-4.0"
  },
  "hardware": {
    "provider": "Google Colab free allocation",
    "gpu": "Tesla T4",
    "gpu_vram_bytes": 15637086208,
    "host_ram_bytes": 13605830656,
    "logical_cpus": 2
  },
  "training_software": {
    "accelerate": "1.8.1",
    "bitsandbytes": "0.46.1",
    "datasets": "3.6.0",
    "huggingface-hub": "0.33.4",
    "numpy": "1.26.4",
    "peft": "0.16.0",
    "psutil": "7.2.2",
    "python": "3.12.13",
    "safetensors": "0.8.0",
    "tokenizers": "0.21.4",
    "torch": "2.7.1+cu126",
    "transformers": "4.53.3",
    "trl": "0.19.1"
  },
  "evaluation_software": {
    "python": "3.12.13 (main, Jun 23 2026, 15:18:55) [Clang 22.1.3 ]",
    "platform": "Linux-6.6.122+-x86_64-with-glibc2.39",
    "packages": {
      "marisa-trie": "1.4.1",
      "catalogue": "2.0.10",
      "cymem": "2.0.13",
      "certifi": "2026.7.22",
      "sniffio": "1.3.1",
      "portalocker": "4.4.0",
      "langcodes": "3.5.0",
      "pytz": "2026.5",
      "sentencepiece": "0.2.2",
      "python-dateutil": "2.9.0.post0",
      "tqdm": "4.70.1",
      "accelerate": "1.8.1",
      "sacremoses": "0.1.1",
      "markdown-it-py": "4.2.0",
      "httpcore": "1.0.9",
      "narwhals": "2.27.1",
      "srsly": "2.5.4",
      "typer": "0.27.3",
      "rouge_score": "0.1.2",
      "nvidia-cusolver-cu12": "11.7.1.2",
      "PyYAML": "6.0.2",
      "nvidia-nvtx-cu12": "12.6.77",
      "nvidia-cusparselt-cu12": "0.6.3",
      "pydantic_core": "2.50.1",
      "filelock": "4.1.1",
      "networkx": "3.7",
      "xxhash": "3.5.0",
      "spacy": "3.8.7",
      "language_data": "1.3.0",
      "absl-py": "2.5.1",
      "sklearn-crfsuite": "0.5.0",
      "fsspec": "2025.3.0",
      "cloudpickle": "3.1.2",
      "stanza": "1.10.1",
      "nvidia-cusparse-cu12": "12.5.4.2",
      "charset-normalizer": "3.5.2",
      "pyvi": "0.1.1",
      "antlr4-python3-runtime": "4.13.2",
      "typepy": "1.3.5",
      "multidict": "7.1.0",
      "aiohappyeyeballs": "2.7.1",
      "spacy-loggers": "1.0.5",
      "nvidia-cuda-nvrtc-cu12": "12.6.77",
      "peft": "0.16.0",
      "Pygments": "2.21.0",
      "attrs": "26.1.0",
      "nvidia-cudnn-cu12": "9.5.1.17",
      "preshed": "3.0.13",
      "annotated-doc": "0.0.5",
      "urllib3": "2.8.0",
      "colorama": "0.4.6",
      "rich": "15.0.0",
      "mpmath": "1.3.0",
      "nvidia-cuda-cupti-cu12": "12.6.80",
      "annotated-types": "0.8.0",
      "wrapt": "2.5.0",
      "anyio": "4.15.1",
      "typer-slim": "0.24.0",
      "gitdb": "4.0.12",
      "packaging": "26.3",
      "confection": "0.1.5",
      "psutil": "7.2.2",
      "nvidia-cublas-cu12": "12.6.4.1",
      "weasel": "0.4.3",
      "blis": "1.3.3",
      "nvidia-cufile-cu12": "1.11.1.6",
      "spacy-legacy": "3.0.12",
      "typing-inspection": "0.4.4",
      "yarl": "1.25.1",
      "click": "8.5.0",
      "propcache": "0.5.4",
      "pandas": "3.0.6",
      "tabulate": "0.10.0",
      "tcolorpy": "0.1.7",
      "typing_extensions": "4.16.0",
      "scikit-learn": "1.9.1",
      "langdetect": "1.0.9",
      "smart_open": "7.7.1",
      "torch": "2.7.1+cu126",
      "nvidia-curand-cu12": "10.3.7.77",
      "chardet": "6.0.0.post1",
      "aiosignal": "1.4.0",
      "pydantic": "2.14.1",
      "shellingham": "1.5.4",
      "aenum": "3.1.15",
      "h11": "0.16.0",
      "nltk": "3.9.1",
      "safetensors": "0.8.0",
      "latex2sympy2_extended": "1.0.6",
      "murmurhash": "1.0.15",
      "mbstrdecoder": "1.1.5",
      "sympy": "1.14.0",
      "lxml": "6.1.3",
      "triton": "3.3.1",
      "mdurl": "0.1.2",
      "aiohttp": "3.14.5",
      "dill": "0.3.8",
      "pyarrow": "20.0.0",
      "nvidia-cufft-cu12": "11.3.0.4",
      "datasets": "3.6.0",
      "six": "1.17.0",
      "transformers": "4.53.3",
      "MarkupSafe": "3.0.4",
      "nvidia-nvjitlink-cu12": "12.6.85",
      "nvidia-nccl-cu12": "2.26.2",
      "cloudpathlib": "0.26.0",
      "pytablewriter": "1.2.1",
      "tzdata": "2026.5",
      "colorlog": "6.12.0",
      "pathvalidate": "3.3.1",
      "hf-xet": "1.7.0",
      "tabledata": "1.3.5",
      "idna": "3.20",
      "lighteval": "0.10.1.dev0",
      "jieba": "0.42.1",
      "thinc": "8.3.11",
      "python-crfsuite": "0.9.12",
      "DataProperty": "1.1.1",
      "smmap": "5.0.3",
      "httpx": "0.27.2",
      "nvidia-cuda-runtime-cu12": "12.6.77",
      "rouge": "1.0.1",
      "protobuf": "7.36.2",
      "GitPython": "3.2.0",
      "regex": "2026.9.29",
      "numpy": "1.26.4",
      "threadpoolctl": "3.7.0",
      "joblib": "1.6.0",
      "tokenizers": "0.21.4",
      "Jinja2": "3.1.6",
      "emoji": "2.16.0",
      "wasabi": "1.1.3",
      "multiprocess": "0.70.16",
      "termcolor": "2.3.0",
      "pycountry": "26.2.16",
      "requests": "2.34.2",
      "huggingface-hub": "0.33.4",
      "setuptools": "84.0.0",
      "scipy": "1.17.1",
      "sacrebleu": "2.6.0",
      "frozenlist": "1.8.0"
    }
  },
  "evaluation": {
    "started_at": "2026-10-12T18:22:26.722768+00:00",
    "finished_at": "2026-10-12T18:54:48.708636+00:00",
    "maximum_input_tokens": 711,
    "peak_allocated_bytes": 7825839616,
    "peak_reserved_bytes": 8241807360,
    "adapter_merged": false,
    "base_rerun": false
  },
  "models": [
    {
      "model": "Qwen/Qwen3-1.7B-Base",
      "correct": 483,
      "samples": 900,
      "accuracy": 0.5366666666666666,
      "accuracy_standard_error": 0.01663103346925096,
      "wall_seconds": 2100.078,
      "inference_seconds": 2067.149676
    },
    {
      "model": "Makata/Makata-v0.1 checkpoint 91",
      "correct": 498,
      "samples": 900,
      "accuracy": 0.5533333333333333,
      "accuracy_standard_error": 0.016580794723924614,
      "wall_seconds": 1941.986,
      "inference_seconds": 1884.626923
    }
  ],
  "paired": {
    "sample_count": 900,
    "base_correct": 483,
    "adapter_correct": 498,
    "difference_percentage_points": 1.6666666666666667,
    "paired_counts": {
      "both_correct": 390,
      "improved": 108,
      "regressed": 93,
      "both_wrong": 309
    },
    "passage_clusters": 488,
    "cluster_sizes": {
      "1": 76,
      "2": 412
    },
    "uncertainty": {
      "method": "paired passage-cluster percentile bootstrap",
      "replicates": 10000,
      "seed": 1234,
      "confidence_level": 0.95,
      "quantile_method": "linear",
      "difference_interval_percentage_points": [
        -1.449315799689441,
        4.804469273743017
      ],
      "limitations": "Conditional on this translated benchmark and one training run; not training-seed or contamination uncertainty."
    }
  },
  "artifacts": {
    "training": {
      "notebook_sha256": "322588b3cd4e93de5a0f0e854aaf5c01402736601fbb9718db1702175b620ce0",
      "source_bundle_sha256": "9e968f19abfb6fa25224e6969042bb87a126d0e7935d882841ee1a47cd1f7bd4"
    },
    "evaluation": {
      "notebook_sha256": "6ff9e4d1acdb5360b6af0a14398be00c11b246e5e71690a106f2c701f148553e",
      "source_bundle_sha256": "d01c97b7d704371a87b1e6357aaf3f0ecfe1e3004c3171a2d7f22a7afb762924"
    }
  },
  "release": {
    "training_data_public": false,
    "checkpoints_public": false,
    "unreviewed_tasks_public": false,
    "unreviewed_tasks_scored": false,
    "qualitative_analysis_complete": false
  }
}