init

2026-07-19 23:52:25 +04:00
commit 99b06b0468
155 changed files with 3808611 additions and 0 deletions
@@ -0,0 +1,14 @@
+from peft import PeftModel
+from transformers import WhisperForConditionalGeneration
+
+base = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small")
+model = PeftModel.from_pretrained(base, "whisper-small/checkpoint-1000")
+
+# check if any lora weights are actually non-zero
+import torch
+for name, param in model.named_parameters():
+    if "lora" in name and param.abs().sum() > 0:
+        print(f"[+] active: {name} | sum: {param.abs().sum().item():.4f}")
+        break
+else:
+    print("[-] no active lora weights found")
@@ -0,0 +1,63 @@
+import torch
+from datasets import load_dataset, Audio
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
+from peft import PeftModel
+import jiwer
+
+MODEL_ID = "openai/whisper-medium"
+BASE_DIR = "whisper-medium-ru-lora"
+LORA_PATH = f"{BASE_DIR}/checkpoint-4000"
+NUM_SAMPLES = 500
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+dtype = torch.float16 if device == "cuda" else torch.float32
+print(f"[*] device: {device}")
+
+processor = WhisperProcessor.from_pretrained(MODEL_ID)
+base_model = WhisperForConditionalGeneration.from_pretrained(MODEL_ID, torch_dtype=dtype).to(device)
+
+model = PeftModel.from_pretrained(base_model, LORA_PATH)
+model.eval()
+
+trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
+total = sum(p.numel() for p in model.parameters())
+print(f"[+] LoRA loaded. trainable: {trainable:,} / {total:,} ({100*trainable/total:.3f}%)")
+
+print("[*] loading common voice ru test split...")
+ds = load_dataset("fsicoli/common_voice_17_0", "ru", split="test", trust_remote_code=True)
+ds = ds.cast_column("audio", Audio(sampling_rate=16000))
+if NUM_SAMPLES:
+    ds = ds.select(range(NUM_SAMPLES))
+
+references, hypotheses = [], []
+
+forced_decoder_ids = processor.get_decoder_prompt_ids(language="russian", task="transcribe")
+
+for i, sample in enumerate(ds):
+    audio = sample["audio"]
+    inputs = processor(
+        audio["array"],
+        sampling_rate=audio["sampling_rate"],
+        return_tensors="pt"
+    )
+    input_features = inputs.input_features.to(device, dtype=dtype)
+
+    model.merge_adapter()
+    with torch.no_grad():
+        predicted_ids = model.base_model.model.generate(
+            input_features=input_features,
+            forced_decoder_ids=forced_decoder_ids,
+        )
+    model.unmerge_adapter()
+
+    text = processor.batch_decode(predicted_ids, skip_special_tokens=True)[0].strip()
+    ref = sample["sentence"].strip()
+    references.append(ref)
+    hypotheses.append(text)
+
+    if i < 5:
+        print(f"  [{i}] ref : {ref!r}")
+        print(f"  [{i}] hyp : {text!r}")
+
+wer = jiwer.wer(references, hypotheses)
+print(f"\n[+] LoRA model WER on {len(ds)} samples: {wer:.4f} ({wer*100:.2f}%)")
@@ -0,0 +1,175 @@
+import sys
+import torch
+import datasets
+from transformers import (
+    AutoProcessor,
+    WhisperForConditionalGeneration,
+    Seq2SeqTrainer,
+    Seq2SeqTrainingArguments
+)
+from peft import LoraConfig, get_peft_model
+from jiwer import wer
+
+datasets.disable_caching()
+
+# === Configuration ===
+DATASET = "fsicoli/common_voice_17_0"
+LANG = "ru"
+SAMPLING_RATE = 16000
+MODEL_NAME = "openai/whisper-small"
+TRAIN_BATCH = 2
+EVAL_BATCH = 2
+MAX_LENGTH = 64
+LR = 1e-4
+MAX_STEPS = 4000
+WARMUP_STEPS = 100
+BATCH_SIZE = 512
+
+print("=== Loading processor", flush=True)
+processor = AutoProcessor.from_pretrained(MODEL_NAME, language=LANG, task="transcribe")
+print("[+] Processor loaded", flush=True)
+
+print("=== Loading model", flush=True)
+model = WhisperForConditionalGeneration.from_pretrained(MODEL_NAME)
+print("[+] Model loaded", flush=True)
+
+print("=== Configuring LoRA", flush=True)
+lora_config = LoraConfig(
+    task_type="SEQ_2_SEQ_LM",
+    target_modules=["q_proj", "k_proj", "v_proj", "out_proj"],
+    r=8,
+    lora_alpha=32,
+    lora_dropout=0.1,
+    bias="none",
+    fan_in_fan_out=False,
+)
+model = get_peft_model(model, lora_config)
+print("[+] LoRA configured", flush=True)
+
+# === Streaming train dataset ===
+print("=== Loading train dataset", flush=True)
+train_ds = datasets.load_dataset(DATASET, LANG, split="train")
+train_ds = train_ds.remove_columns(["accent", "age", "client_id", "down_votes", "gender", "locale", "path", "segment", "up_votes"])
+print("[+] Train dataset loaded", flush=True)
+
+# === Small cached validation/test subsets ===
+print("=== Loading validation/test datasets", flush=True)
+val_ds = datasets.load_dataset(DATASET, LANG, split="validation[:500]")
+val_ds = val_ds.remove_columns(["accent", "age", "client_id", "down_votes", "gender", "locale", "path", "segment", "up_votes"])
+test_ds = datasets.load_dataset(DATASET, LANG, split="test[:500]")
+test_ds = test_ds.remove_columns(["accent", "age", "client_id", "down_votes", "gender", "locale", "path", "segment", "up_votes"])
+print("[+] Validation/test datasets are loaded", flush=True)
+
+# Cast audio column
+print("=== Casting datasets to 16kHz", flush=True)
+train_ds = train_ds.cast_column("audio", datasets.Audio(sampling_rate=SAMPLING_RATE))
+val_ds = val_ds.cast_column("audio", datasets.Audio(sampling_rate=SAMPLING_RATE))
+test_ds = test_ds.cast_column("audio", datasets.Audio(sampling_rate=SAMPLING_RATE))
+print("[+] Audio column cast to 16kHz", flush=True)
+
+# === Map function with picklable args ===
+def prepare_examples(batch, processor_name=MODEL_NAME):
+    texts = batch["sentence"]
+    audios = [x["array"] for x in batch["audio"]]
+    inputs = processor(
+        audios,
+        sampling_rate=SAMPLING_RATE,
+        return_tensors="np"
+    )
+    batch["input_features"] = inputs.input_features
+
+    batch["labels"] = processor.tokenizer(
+        texts,
+        padding="max_length",
+        truncation=True,
+        max_length=MAX_LENGTH
+    ).input_ids
+    return batch
+
+# === Preprocessing ===
+try:
+    print("=== Preprocessing datasets", flush=True)
+    train_ds = train_ds.map(
+        prepare_examples,
+        batched=True,
+        batch_size=BATCH_SIZE,
+        remove_columns=["audio", "sentence"],
+        load_from_cache_file=False,
+    )
+    val_ds = val_ds.map(
+        prepare_examples,
+        batched=True,
+        batch_size=BATCH_SIZE,
+        remove_columns=["audio", "sentence"],
+        load_from_cache_file=False,
+    )
+    test_ds = test_ds.map(
+        prepare_examples,
+        batched=True,
+        batch_size=BATCH_SIZE,
+        remove_columns=["audio", "sentence"],
+        load_from_cache_file=False,
+    )
+    print("[+] Datasets are preprocessed", flush=True)
+except Exception as e:
+    print(f"[-] Preprocessing failed: {e}", flush=True)
+    sys.exit(1)
+
+# === Data collator ===
+def data_collator(batch):
+    input_features = torch.tensor([ex["input_features"] for ex in batch], dtype=torch.float32)
+    labels = torch.tensor([ex["labels"] for ex in batch], dtype=torch.long)
+    return {"input_features": input_features, "labels": labels}
+
+# === Metrics ===
+def compute_metrics(pred):
+    pred_ids = pred.predictions
+    label_ids = pred.label_ids
+    pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)
+    label_str = processor.batch_decode(label_ids, skip_special_tokens=True)
+    return {"wer": wer(label_str, pred_str)}
+
+# === Training arguments ===
+training_args = Seq2SeqTrainingArguments(
+    output_dir="./whisper_lora",
+    per_device_train_batch_size=TRAIN_BATCH,
+    gradient_accumulation_steps=1,
+    learning_rate=LR,
+    warmup_steps=WARMUP_STEPS,
+    max_steps=MAX_STEPS,
+    gradient_checkpointing=True,
+    predict_with_generate=True,
+    per_device_eval_batch_size=EVAL_BATCH,
+    generation_max_length=MAX_LENGTH,
+    save_steps=1000,
+    logging_steps=100,
+    fp16=True,
+    eval_strategy="steps",
+    eval_steps=1000,
+    metric_for_best_model="wer",
+    greater_is_better=False,
+    load_best_model_at_end=True,
+    report_to=["tensorboard"],
+)
+
+# === Trainer ===
+print("=== Initializing trainer", flush=True)
+trainer = Seq2SeqTrainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_ds,
+    eval_dataset=val_ds,
+    data_collator=data_collator,
+    compute_metrics=compute_metrics,
+)
+print("[+] Trainer initialized", flush=True)
+
+# === Start training ===
+print("=== Starting training", flush=True)
+trainer.train()
+print("[+] Training finished", flush=True)
+
+# === Final evaluation on test set ===
+print("=== Evaluating on test set", flush=True)
+results = trainer.evaluate(test_ds)
+print(f"[+] Test WER: {results['eval_wer']:.4f}", flush=True)
@@ -0,0 +1,53 @@
+import torch
+from datasets import load_dataset, Audio
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
+import jiwer
+
+MODEL_ID = "openai/whisper-medium"
+NUM_SAMPLES = 50  # how many to eval, None = full set
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+dtype = torch.float16 if device == "cuda" else torch.float32
+print(f"[*] device: {device}")
+
+processor = WhisperProcessor.from_pretrained(MODEL_ID)
+model = WhisperForConditionalGeneration.from_pretrained(MODEL_ID, torch_dtype=dtype).to(device)
+model.eval()
+print(f"[+] base model loaded. params: {sum(p.numel() for p in model.parameters()):,}")
+
+print("[*] loading common voice ru test split...")
+ds = load_dataset("fsicoli/common_voice_17_0", "ru", split="test", trust_remote_code=True)
+ds = ds.cast_column("audio", Audio(sampling_rate=16000))
+if NUM_SAMPLES:
+    ds = ds.select(range(NUM_SAMPLES))
+
+references, hypotheses = [], []
+
+forced_decoder_ids = processor.get_decoder_prompt_ids(language="russian", task="transcribe")
+
+for i, sample in enumerate(ds):
+    audio = sample["audio"]
+    inputs = processor(
+        audio["array"],
+        sampling_rate=audio["sampling_rate"],
+        return_tensors="pt"
+    )
+    input_features = inputs.input_features.to(device, dtype=dtype)
+
+    with torch.no_grad():
+        predicted_ids = model.generate(
+            input_features,
+            forced_decoder_ids=forced_decoder_ids,
+        )
+
+    text = processor.batch_decode(predicted_ids, skip_special_tokens=True)[0].strip()
+    ref = sample["sentence"].strip()
+    references.append(ref)
+    hypotheses.append(text)
+
+    if i < 5:  # print first few so you see something's happening
+        print(f"  [{i}] ref : {ref!r}")
+        print(f"  [{i}] hyp : {text!r}")
+
+wer = jiwer.wer(references, hypotheses)
+print(f"\n[+] base model WER on {len(ds)} samples: {wer:.4f} ({wer*100:.2f}%)")
@@ -0,0 +1,323 @@
+import os
+import sys
+
+os.environ.setdefault("HF_HOME", "/mnt/D/.cache/huggingface")
+os.environ.setdefault("HF_DATASETS_CACHE", "/mnt/D/.cache/huggingface/datasets")
+os.environ.setdefault("TMPDIR", "/mnt/D/tmp")
+
+os.makedirs(os.environ["TMPDIR"], exist_ok=True)
+os.makedirs(os.environ["HF_DATASETS_CACHE"], exist_ok=True)
+
+print("=== initializing environment")
+
+from dataclasses import dataclass
+from typing import Any
+
+import jiwer
+from datasets import load_dataset, Audio
+from peft import LoraConfig, TaskType, get_peft_model
+import numpy as np
+import torch
+from tqdm import tqdm as _tqdm
+from transformers import (
+    Seq2SeqTrainer,
+    Seq2SeqTrainingArguments,
+    WhisperForConditionalGeneration,
+    WhisperProcessor,
+)
+
+
+class WhisperLoraTrainer(Seq2SeqTrainer):
+    def compute_loss(self, model, inputs, return_outputs=False, **kwargs):
+        input_features = inputs.get("input_features")
+        labels = inputs.get("labels")
+        dtype = next(model.parameters()).dtype
+        input_features = input_features.to(dtype)
+        labels = labels.to(torch.long)
+        outputs = model.base_model(input_features=input_features, labels=labels)
+        loss = outputs.loss
+        return (loss, outputs) if return_outputs else loss
+
+    def prediction_step(self, model, inputs, prediction_loss_only, ignore_keys=None):
+        if not self.args.predict_with_generate or prediction_loss_only:
+            return super().prediction_step(
+                model, inputs, prediction_loss_only, ignore_keys=ignore_keys
+            )
+
+        inputs = self._prepare_inputs(inputs)
+        labels = inputs.get("labels")
+        dtype = next(model.parameters()).dtype
+
+        with torch.no_grad():
+            generated_tokens = model.base_model.model.generate(
+                input_features=inputs["input_features"].to(dtype),
+                language=LANGUAGE,
+                task="transcribe",
+            )
+
+        loss = None
+        if labels is not None:
+            with torch.no_grad():
+                outputs = model.base_model(
+                    input_features=inputs["input_features"].to(dtype),
+                    labels=labels.to(torch.long),
+                )
+            loss = outputs.loss.detach().float()
+
+        generated_tokens = generated_tokens.detach().cpu().to(torch.long)
+        if labels is not None:
+            labels = labels.detach().cpu().to(torch.long)
+
+        return loss, generated_tokens, labels
+
+print("[+] imports ok")
+
+
+MODEL_ID = "openai/whisper-medium"
+DATASET_ID = "fsicoli/common_voice_17_0"
+LANGUAGE = "ru"
+SAMPLING_RATE = 16000
+OUTPUT_DIR = "./whisper-medium-ru-lora"
+EVAL_SAMPLES = 1000
+LORA_R = 32
+LORA_ALPHA = 64
+LORA_DROPOUT = 0.05
+
+
+@dataclass
+class DataCollatorSpeechSeq2SeqWithPadding:
+    processor: Any
+    decoder_start_token_id: int
+
+    def __call__(self, features: list[dict]) -> dict:
+        input_features = [{"input_features": f["input_features"]} for f in features]
+        batch = self.processor.feature_extractor.pad(input_features, return_tensors="pt")
+
+        label_features = [{"input_ids": f["labels"]} for f in features]
+        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors="pt")
+
+        labels = labels_batch["input_ids"].masked_fill(
+            labels_batch.attention_mask.ne(1), -100
+        )
+
+        if (labels[:, 0] == self.decoder_start_token_id).all().cpu().item():
+            labels = labels[:, 1:]
+
+        batch["labels"] = labels
+        return batch
+
+
+def load_processor_and_model():
+    print("=== loading processor")
+    try:
+        processor = WhisperProcessor.from_pretrained(
+            MODEL_ID, language=LANGUAGE, task="transcribe"
+        )
+        print("[+] processor loaded")
+    except Exception as e:
+        print(f"[-] processor load failed: {e}")
+        sys.exit(1)
+
+    print("=== loading model")
+    try:
+        model = WhisperForConditionalGeneration.from_pretrained(MODEL_ID)
+        model.config.use_cache = False
+        model.config.forced_decoder_ids = None
+        model.config.suppress_tokens = []
+        model.generation_config.language = LANGUAGE
+        model.generation_config.task = "transcribe"
+        model.generation_config.forced_decoder_ids = None
+        model.generation_config.max_length = None
+        print("[+] model loaded")
+    except Exception as e:
+        print(f"[-] model load failed: {e}")
+        sys.exit(1)
+
+    return processor, model
+
+
+def apply_lora(model):
+    print("=== applying LoRA")
+    try:
+        lora_config = LoraConfig(
+            r=LORA_R,
+            lora_alpha=LORA_ALPHA,
+            lora_dropout=LORA_DROPOUT,
+            target_modules=["q_proj", "k_proj", "v_proj", "out_proj"],
+            task_type=TaskType.SEQ_2_SEQ_LM,
+        )
+        model = get_peft_model(model, lora_config)
+        model.enable_input_require_grads()
+        model.print_trainable_parameters()
+        print("[+] LoRA applied")
+    except Exception as e:
+        print(f"[-] LoRA apply failed: {e}")
+        sys.exit(1)
+    return model
+
+
+def prepare_batch(batch, processor):
+    audio = batch["audio"]
+    batch["input_features"] = processor.feature_extractor(
+        audio["array"], sampling_rate=audio["sampling_rate"]
+    ).input_features[0]
+    batch["labels"] = processor.tokenizer(batch["sentence"]).input_ids
+    return batch
+
+
+def load_and_prepare_dataset(processor):
+    print("=== loading dataset (train + validation only)")
+    try:
+        raw_train, raw_eval = load_dataset(
+            DATASET_ID,
+            LANGUAGE,
+            split=["train", "validation"],
+            trust_remote_code=True,
+        )
+        print("[+] dataset loaded")
+    except Exception as e:
+        print(f"[-] dataset load failed: {e}")
+        sys.exit(1)
+
+    print(f"=== slicing validation to {EVAL_SAMPLES} samples")
+    raw_eval = raw_eval.select(range(EVAL_SAMPLES))
+    print(f"[+] validation sliced to {len(raw_eval)} samples")
+
+    print("=== casting audio column")
+    try:
+        raw_train = raw_train.cast_column("audio", Audio(sampling_rate=SAMPLING_RATE))
+        raw_eval = raw_eval.cast_column("audio", Audio(sampling_rate=SAMPLING_RATE))
+        print("[+] audio cast done")
+    except Exception as e:
+        print(f"[-] audio cast failed: {e}")
+        sys.exit(1)
+
+    fn = lambda batch: prepare_batch(batch, processor)
+
+    print("=== mapping train split")
+    try:
+        train_dataset = raw_train.map(
+            fn,
+            remove_columns=raw_train.column_names,
+            num_proc=1,
+            cache_file_name=str(os.path.join(os.environ["HF_DATASETS_CACHE"], "cv17_ru_train.arrow")),
+        )
+        print("[+] train split mapped")
+    except Exception as e:
+        print(f"[-] train map failed: {e}")
+        sys.exit(1)
+
+    print("=== mapping validation split")
+    try:
+        eval_dataset = raw_eval.map(
+            fn,
+            remove_columns=raw_eval.column_names,
+            num_proc=1,
+            load_from_cache_file=False,
+        )
+        print("[+] validation split mapped")
+    except Exception as e:
+        print(f"[-] validation map failed: {e}")
+        sys.exit(1)
+
+    return train_dataset, eval_dataset
+
+
+def build_compute_metrics(processor):
+    def compute_metrics(pred):
+        pred_ids = pred.predictions
+        label_ids = pred.label_ids
+
+        if isinstance(pred_ids, tuple):
+            pred_ids = pred_ids[0]
+
+        label_ids = np.array(label_ids, dtype=np.int64)
+        label_ids[label_ids == -100] = processor.tokenizer.pad_token_id
+
+        pred_str = [
+            processor.tokenizer.decode(
+                [int(t) for t in seq if 0 <= int(t) < processor.tokenizer.vocab_size],
+                skip_special_tokens=True,
+            )
+            for seq in pred_ids
+        ]
+        label_str = processor.tokenizer.batch_decode(label_ids, skip_special_tokens=True)
+
+        wer = 100 * jiwer.wer(label_str, pred_str)
+        _tqdm.write(f"[+] eval wer: {wer:.2f}%")
+        return {"wer": wer}
+
+    return compute_metrics
+
+
+def main():
+    processor, model = load_processor_and_model()
+    model = apply_lora(model)
+    train_dataset, eval_dataset = load_and_prepare_dataset(processor)
+
+    data_collator = DataCollatorSpeechSeq2SeqWithPadding(
+        processor=processor,
+        decoder_start_token_id=model.config.decoder_start_token_id,
+    )
+    print("[+] data collator configured")
+
+    compute_metrics = build_compute_metrics(processor)
+
+    print("=== configuring training arguments")
+    training_args = Seq2SeqTrainingArguments(
+        output_dir=OUTPUT_DIR,
+        per_device_train_batch_size=2,
+        gradient_accumulation_steps=1,
+        learning_rate=5e-5,
+        warmup_steps=500,
+        max_steps=4000,
+        gradient_checkpointing=True,
+        predict_with_generate=True,
+        per_device_eval_batch_size=2,
+        generation_max_length=64,
+        save_steps=1000,
+        logging_steps=100,
+        fp16=True,
+        fp16_full_eval=False,
+        eval_strategy="steps",
+        eval_steps=1000,
+        metric_for_best_model="wer",
+        greater_is_better=False,
+        load_best_model_at_end=True,
+        generation_config=None,
+        report_to=["tensorboard"],
+    )
+    print("[+] training arguments configured")
+
+    print("=== initializing trainer")
+    try:
+        trainer = WhisperLoraTrainer(
+            model=model,
+            args=training_args,
+            train_dataset=train_dataset,
+            eval_dataset=eval_dataset,
+            data_collator=data_collator,
+            compute_metrics=compute_metrics,
+            processing_class=processor.feature_extractor,
+        )
+        print("[+] trainer initialized")
+    except Exception as e:
+        print(f"[-] trainer init failed: {e}")
+        sys.exit(1)
+
+    print("=== starting training")
+    trainer.train()
+    print("[+] training complete")
+
+    print("=== saving model")
+    try:
+        model.save_pretrained(OUTPUT_DIR)
+        processor.save_pretrained(OUTPUT_DIR)
+        print(f"[+] model saved to {OUTPUT_DIR}")
+    except Exception as e:
+        print(f"[-] save failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,206 @@
+---
+base_model: openai/whisper-medium
+library_name: peft
+tags:
+- base_model:adapter:openai/whisper-medium
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "openai/whisper-medium",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "out_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "SEQ_2_SEQ_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
@@ -0,0 +1,17 @@
+{
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "WhisperProcessor"
+}
@@ -0,0 +1,127 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|endoftext|>",
+    "<|startoftranscript|>",
+    "<|en|>",
+    "<|zh|>",
+    "<|de|>",
+    "<|es|>",
+    "<|ru|>",
+    "<|ko|>",
+    "<|fr|>",
+    "<|ja|>",
+    "<|pt|>",
+    "<|tr|>",
+    "<|pl|>",
+    "<|ca|>",
+    "<|nl|>",
+    "<|ar|>",
+    "<|sv|>",
+    "<|it|>",
+    "<|id|>",
+    "<|hi|>",
+    "<|fi|>",
+    "<|vi|>",
+    "<|he|>",
+    "<|uk|>",
+    "<|el|>",
+    "<|ms|>",
+    "<|cs|>",
+    "<|ro|>",
+    "<|da|>",
+    "<|hu|>",
+    "<|ta|>",
+    "<|no|>",
+    "<|th|>",
+    "<|ur|>",
+    "<|hr|>",
+    "<|bg|>",
+    "<|lt|>",
+    "<|la|>",
+    "<|mi|>",
+    "<|ml|>",
+    "<|cy|>",
+    "<|sk|>",
+    "<|te|>",
+    "<|fa|>",
+    "<|lv|>",
+    "<|bn|>",
+    "<|sr|>",
+    "<|az|>",
+    "<|sl|>",
+    "<|kn|>",
+    "<|et|>",
+    "<|mk|>",
+    "<|br|>",
+    "<|eu|>",
+    "<|is|>",
+    "<|hy|>",
+    "<|ne|>",
+    "<|mn|>",
+    "<|bs|>",
+    "<|kk|>",
+    "<|sq|>",
+    "<|sw|>",
+    "<|gl|>",
+    "<|mr|>",
+    "<|pa|>",
+    "<|si|>",
+    "<|km|>",
+    "<|sn|>",
+    "<|yo|>",
+    "<|so|>",
+    "<|af|>",
+    "<|oc|>",
+    "<|ka|>",
+    "<|be|>",
+    "<|tg|>",
+    "<|sd|>",
+    "<|gu|>",
+    "<|am|>",
+    "<|yi|>",
+    "<|lo|>",
+    "<|uz|>",
+    "<|fo|>",
+    "<|ht|>",
+    "<|ps|>",
+    "<|tk|>",
+    "<|nn|>",
+    "<|mt|>",
+    "<|sa|>",
+    "<|lb|>",
+    "<|my|>",
+    "<|bo|>",
+    "<|tl|>",
+    "<|mg|>",
+    "<|as|>",
+    "<|tt|>",
+    "<|haw|>",
+    "<|ln|>",
+    "<|ha|>",
+    "<|ba|>",
+    "<|jw|>",
+    "<|su|>",
+    "<|translate|>",
+    "<|transcribe|>",
+    "<|startoflm|>",
+    "<|startofprev|>",
+    "<|nocaptions|>",
+    "<|notimestamps|>"
+  ],
+  "is_local": false,
+  "language": "ru",
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "predict_timestamps": false,
+  "processor_class": "WhisperProcessor",
+  "return_attention_mask": false,
+  "task": "transcribe",
+  "tokenizer_class": "WhisperTokenizer",
+  "unk_token": "<|endoftext|>"
+}
@@ -0,0 +1,206 @@
+---
+base_model: openai/whisper-small
+library_name: peft
+tags:
+- base_model:adapter:openai/whisper-small
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "openai/whisper-small",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "out_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "SEQ_2_SEQ_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}
@@ -0,0 +1,17 @@
+{
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "WhisperProcessor"
+}
@@ -0,0 +1,127 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|endoftext|>",
+    "<|startoftranscript|>",
+    "<|en|>",
+    "<|zh|>",
+    "<|de|>",
+    "<|es|>",
+    "<|ru|>",
+    "<|ko|>",
+    "<|fr|>",
+    "<|ja|>",
+    "<|pt|>",
+    "<|tr|>",
+    "<|pl|>",
+    "<|ca|>",
+    "<|nl|>",
+    "<|ar|>",
+    "<|sv|>",
+    "<|it|>",
+    "<|id|>",
+    "<|hi|>",
+    "<|fi|>",
+    "<|vi|>",
+    "<|he|>",
+    "<|uk|>",
+    "<|el|>",
+    "<|ms|>",
+    "<|cs|>",
+    "<|ro|>",
+    "<|da|>",
+    "<|hu|>",
+    "<|ta|>",
+    "<|no|>",
+    "<|th|>",
+    "<|ur|>",
+    "<|hr|>",
+    "<|bg|>",
+    "<|lt|>",
+    "<|la|>",
+    "<|mi|>",
+    "<|ml|>",
+    "<|cy|>",
+    "<|sk|>",
+    "<|te|>",
+    "<|fa|>",
+    "<|lv|>",
+    "<|bn|>",
+    "<|sr|>",
+    "<|az|>",
+    "<|sl|>",
+    "<|kn|>",
+    "<|et|>",
+    "<|mk|>",
+    "<|br|>",
+    "<|eu|>",
+    "<|is|>",
+    "<|hy|>",
+    "<|ne|>",
+    "<|mn|>",
+    "<|bs|>",
+    "<|kk|>",
+    "<|sq|>",
+    "<|sw|>",
+    "<|gl|>",
+    "<|mr|>",
+    "<|pa|>",
+    "<|si|>",
+    "<|km|>",
+    "<|sn|>",
+    "<|yo|>",
+    "<|so|>",
+    "<|af|>",
+    "<|oc|>",
+    "<|ka|>",
+    "<|be|>",
+    "<|tg|>",
+    "<|sd|>",
+    "<|gu|>",
+    "<|am|>",
+    "<|yi|>",
+    "<|lo|>",
+    "<|uz|>",
+    "<|fo|>",
+    "<|ht|>",
+    "<|ps|>",
+    "<|tk|>",
+    "<|nn|>",
+    "<|mt|>",
+    "<|sa|>",
+    "<|lb|>",
+    "<|my|>",
+    "<|bo|>",
+    "<|tl|>",
+    "<|mg|>",
+    "<|as|>",
+    "<|tt|>",
+    "<|haw|>",
+    "<|ln|>",
+    "<|ha|>",
+    "<|ba|>",
+    "<|jw|>",
+    "<|su|>",
+    "<|translate|>",
+    "<|transcribe|>",
+    "<|startoflm|>",
+    "<|startofprev|>",
+    "<|nocaptions|>",
+    "<|notimestamps|>"
+  ],
+  "is_local": false,
+  "language": "ru",
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "predict_timestamps": false,
+  "processor_class": "WhisperProcessor",
+  "return_attention_mask": false,
+  "task": "transcribe",
+  "tokenizer_class": "WhisperTokenizer",
+  "unk_token": "<|endoftext|>"
+}