From a3aa5bf849b3b7150df8fc84172870077cba03b8 Mon Sep 17 00:00:00 2001 From: kimura-keiji Date: Thu, 6 Aug 2026 22:52:53 +0900 Subject: [PATCH 1/2] Define v1-3-2 --- CHANGELOG.md | 2 ++ onecomp/__version__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 556995d1..85119d87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Change log +## [v1.3.2] 2026-08-dd + ## [v1.3.1] 2026-08-06 ### Bug Fix diff --git a/onecomp/__version__.py b/onecomp/__version__.py index f21c7223..0919865f 100644 --- a/onecomp/__version__.py +++ b/onecomp/__version__.py @@ -6,4 +6,4 @@ """ -__version__ = "1.3.1" +__version__ = "1.3.2" From 34facf3cdd24b09f46d0833f2f67c27197e579cf Mon Sep 17 00:00:00 2001 From: katari Date: Tue, 18 Aug 2026 17:40:20 +0900 Subject: [PATCH 2/2] [fix] Use canonical WikiText dataset ID --- CHANGELOG.md | 7 +++++++ docs/user-guide/examples.md | 2 +- docs/user-guide/post-process.md | 8 ++++---- example/post_process/example_lora_sft.py | 2 +- onecomp/runner.py | 4 ++-- onecomp/utils/perplexity.py | 4 ++-- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 556995d1..4e705d84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Change log +## [v1.3.2(WIP)+fix/wikitext-hf-dataset-id] 2026-08-18 + +### Bug Fix + +- Fix WikiText dataset loading in clean environments by using the canonical + `Salesforce/wikitext` dataset ID for perplexity evaluation and LoRA SFT examples. + ## [v1.3.1] 2026-08-06 ### Bug Fix diff --git a/docs/user-guide/examples.md b/docs/user-guide/examples.md index fb988aed..cc7aee89 100644 --- a/docs/user-guide/examples.md +++ b/docs/user-guide/examples.md @@ -578,7 +578,7 @@ model_config = ModelConfig( gptq = GPTQ(wbits=4, groupsize=128) post_process = PostProcessLoraSFT( - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config_name="wikitext-2-raw-v1", train_split="train", text_column="text", diff --git a/docs/user-guide/post-process.md b/docs/user-guide/post-process.md index 45459a38..4dca6376 100644 --- a/docs/user-guide/post-process.md +++ b/docs/user-guide/post-process.md @@ -354,7 +354,7 @@ model_config = ModelConfig( gptq = GPTQ(wbits=4, groupsize=128) post_process = PostProcessLoraSFT( - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config_name="wikitext-2-raw-v1", train_split="train", text_column="text", @@ -512,7 +512,7 @@ model, tokenizer = load_quantized_model_pt( ```python PostProcessLoraSFT( - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config_name="wikitext-2-raw-v1", train_split="train", text_column="text", @@ -560,7 +560,7 @@ Teacher distillation aligns the quantized model's output distribution with a ful ```python post_process = PostProcessLoraSFT( - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config_name="wikitext-2-raw-v1", train_split="train", text_column="text", @@ -591,7 +591,7 @@ Intermediate block alignment adds a loss term that aligns hidden states at selec ```python post_process = PostProcessLoraSFT( - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config_name="wikitext-2-raw-v1", train_split="train", text_column="text", diff --git a/example/post_process/example_lora_sft.py b/example/post_process/example_lora_sft.py index 1bf962eb..03d0c7a4 100644 --- a/example/post_process/example_lora_sft.py +++ b/example/post_process/example_lora_sft.py @@ -66,7 +66,7 @@ def generate_text(model, tokenizer, prompt, device, max_new_tokens=64): gptq = GPTQ(wbits=4, groupsize=128) post_process = PostProcessLoraSFT( - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config_name="wikitext-2-raw-v1", train_split="train", text_column="text", diff --git a/onecomp/runner.py b/onecomp/runner.py index b869ef3a..2ffa41d8 100644 --- a/onecomp/runner.py +++ b/onecomp/runner.py @@ -1260,7 +1260,7 @@ def calculate_perplexity( original_model=False, dequantized_model=False, quantized_model=True, - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config="wikitext-2-raw-v1", split="test", max_samples=None, @@ -1342,7 +1342,7 @@ def benchmark_perplexity( original_model=True, dequantized_model=False, quantized_model=True, - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config="wikitext-2-raw-v1", split="test", max_samples=None, diff --git a/onecomp/utils/perplexity.py b/onecomp/utils/perplexity.py index 8c8742b3..d26c701f 100644 --- a/onecomp/utils/perplexity.py +++ b/onecomp/utils/perplexity.py @@ -41,7 +41,7 @@ def calculate_perplexity( model=None, tokenizer=None, model_config=None, - dataset_name="wikitext", + dataset_name="Salesforce/wikitext", dataset_config="wikitext-2-raw-v1", split="test", max_samples=None, @@ -53,7 +53,7 @@ def calculate_perplexity( Based on https://huggingface.co/docs/transformers/perplexity Args: - dataset_name (str): Dataset name (e.g. "wikitext", "allenai/c4"). + dataset_name (str): Dataset name (e.g. "Salesforce/wikitext", "allenai/c4"). dataset_config (str): Dataset configuration. - For WikiText: "wikitext-2-raw-v1" - For C4: "en/c4-train.00001-of-01024.json.gz" (treated as data_files)