Whisper_ko_finetune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f667bf93de0463781f35057530bfc9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shandy77/.local/lib/python3.10/site-packages/datasets/load.py:1454: FutureWarning: The repository for mozilla-foundation/common_voice_13_0 contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/mozilla-foundation/common_voice_13_0\n",
      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
      "  warnings.warn(\n",
      "/home/shandy77/.local/lib/python3.10/site-packages/datasets/load.py:1454: FutureWarning: The repository for mozilla-foundation/common_voice_13_0 contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/mozilla-foundation/common_voice_13_0\n",
      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset, DatasetDict\n",
    "\n",
    "common_voice = DatasetDict()\n",
    "\n",
    "common_voice[\"train\"] = load_dataset(\n",
    "    \"mozilla-foundation/common_voice_13_0\", \"ko\", split=\"train+validation\"\n",
    ")\n",
    "common_voice[\"test\"] = load_dataset(\n",
    "    \"mozilla-foundation/common_voice_13_0\", \"ko\", split=\"test\"\n",
    ")\n",
    "\n",
    "print(common_voice)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_voice = common_voice.select_columns([\"audio\", \"sentence\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'english': 'en',\n",
       " 'chinese': 'zh',\n",
       " 'german': 'de',\n",
       " 'spanish': 'es',\n",
       " 'russian': 'ru',\n",
       " 'korean': 'ko',\n",
       " 'french': 'fr',\n",
       " 'japanese': 'ja',\n",
       " 'portuguese': 'pt',\n",
       " 'turkish': 'tr',\n",
       " 'polish': 'pl',\n",
       " 'catalan': 'ca',\n",
       " 'dutch': 'nl',\n",
       " 'arabic': 'ar',\n",
       " 'swedish': 'sv',\n",
       " 'italian': 'it',\n",
       " 'indonesian': 'id',\n",
       " 'hindi': 'hi',\n",
       " 'finnish': 'fi',\n",
       " 'vietnamese': 'vi',\n",
       " 'hebrew': 'he',\n",
       " 'ukrainian': 'uk',\n",
       " 'greek': 'el',\n",
       " 'malay': 'ms',\n",
       " 'czech': 'cs',\n",
       " 'romanian': 'ro',\n",
       " 'danish': 'da',\n",
       " 'hungarian': 'hu',\n",
       " 'tamil': 'ta',\n",
       " 'norwegian': 'no',\n",
       " 'thai': 'th',\n",
       " 'urdu': 'ur',\n",
       " 'croatian': 'hr',\n",
       " 'bulgarian': 'bg',\n",
       " 'lithuanian': 'lt',\n",
       " 'latin': 'la',\n",
       " 'maori': 'mi',\n",
       " 'malayalam': 'ml',\n",
       " 'welsh': 'cy',\n",
       " 'slovak': 'sk',\n",
       " 'telugu': 'te',\n",
       " 'persian': 'fa',\n",
       " 'latvian': 'lv',\n",
       " 'bengali': 'bn',\n",
       " 'serbian': 'sr',\n",
       " 'azerbaijani': 'az',\n",
       " 'slovenian': 'sl',\n",
       " 'kannada': 'kn',\n",
       " 'estonian': 'et',\n",
       " 'macedonian': 'mk',\n",
       " 'breton': 'br',\n",
       " 'basque': 'eu',\n",
       " 'icelandic': 'is',\n",
       " 'armenian': 'hy',\n",
       " 'nepali': 'ne',\n",
       " 'mongolian': 'mn',\n",
       " 'bosnian': 'bs',\n",
       " 'kazakh': 'kk',\n",
       " 'albanian': 'sq',\n",
       " 'swahili': 'sw',\n",
       " 'galician': 'gl',\n",
       " 'marathi': 'mr',\n",
       " 'punjabi': 'pa',\n",
       " 'sinhala': 'si',\n",
       " 'khmer': 'km',\n",
       " 'shona': 'sn',\n",
       " 'yoruba': 'yo',\n",
       " 'somali': 'so',\n",
       " 'afrikaans': 'af',\n",
       " 'occitan': 'oc',\n",
       " 'georgian': 'ka',\n",
       " 'belarusian': 'be',\n",
       " 'tajik': 'tg',\n",
       " 'sindhi': 'sd',\n",
       " 'gujarati': 'gu',\n",
       " 'amharic': 'am',\n",
       " 'yiddish': 'yi',\n",
       " 'lao': 'lo',\n",
       " 'uzbek': 'uz',\n",
       " 'faroese': 'fo',\n",
       " 'haitian creole': 'ht',\n",
       " 'pashto': 'ps',\n",
       " 'turkmen': 'tk',\n",
       " 'nynorsk': 'nn',\n",
       " 'maltese': 'mt',\n",
       " 'sanskrit': 'sa',\n",
       " 'luxembourgish': 'lb',\n",
       " 'myanmar': 'my',\n",
       " 'tibetan': 'bo',\n",
       " 'tagalog': 'tl',\n",
       " 'malagasy': 'mg',\n",
       " 'assamese': 'as',\n",
       " 'tatar': 'tt',\n",
       " 'hawaiian': 'haw',\n",
       " 'lingala': 'ln',\n",
       " 'hausa': 'ha',\n",
       " 'bashkir': 'ba',\n",
       " 'javanese': 'jw',\n",
       " 'sundanese': 'su',\n",
       " 'cantonese': 'yue',\n",
       " 'burmese': 'my',\n",
       " 'valencian': 'ca',\n",
       " 'flemish': 'nl',\n",
       " 'haitian': 'ht',\n",
       " 'letzeburgesch': 'lb',\n",
       " 'pushto': 'ps',\n",
       " 'panjabi': 'pa',\n",
       " 'moldavian': 'ro',\n",
       " 'moldovan': 'ro',\n",
       " 'sinhalese': 'si',\n",
       " 'castilian': 'es',\n",
       " 'mandarin': 'zh'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers.models.whisper.tokenization_whisper import TO_LANGUAGE_CODE\n",
    "\n",
    "TO_LANGUAGE_CODE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import WhisperProcessor\n",
    "\n",
    "processor = WhisperProcessor.from_pretrained(\n",
    "    \"openai/whisper-small\", language=\"korean\", task=\"transcribe\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'audio': Audio(sampling_rate=48000, mono=True, decode=True, id=None),\n",
       " 'sentence': Value(dtype='string', id=None)}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_voice[\"train\"].features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Audio\n",
    "\n",
    "sampling_rate = processor.feature_extractor.sampling_rate\n",
    "common_voice = common_voice.cast_column(\"audio\", Audio(sampling_rate=sampling_rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_dataset(example):\n",
    "    audio = example[\"audio\"]\n",
    "\n",
    "    example = processor(\n",
    "        audio=audio[\"array\"],\n",
    "        sampling_rate=audio[\"sampling_rate\"],\n",
    "        text=example[\"sentence\"],\n",
    "    )\n",
    "\n",
    "    # compute input length of audio sample in seconds\n",
    "    example[\"input_length\"] = len(audio[\"array\"]) / audio[\"sampling_rate\"]\n",
    "\n",
    "    return example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_voice = common_voice.map(\n",
    "    prepare_dataset, remove_columns=common_voice.column_names[\"train\"], num_proc=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_input_length = 30.0\n",
    "\n",
    "\n",
    "def is_audio_in_length_range(length):\n",
    "    return length < max_input_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_voice[\"train\"] = common_voice[\"train\"].filter(\n",
    "    is_audio_in_length_range,\n",
    "    input_columns=[\"input_length\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_features', 'labels', 'input_length'],\n",
       "    num_rows: 297\n",
       "})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_voice[\"train\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from dataclasses import dataclass\n",
    "from typing import Any, Dict, List, Union\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class DataCollatorSpeechSeq2SeqWithPadding:\n",
    "    processor: Any\n",
    "\n",
    "    def __call__(\n",
    "        self, features: List[Dict[str, Union[List[int], torch.Tensor]]]\n",
    "    ) -> Dict[str, torch.Tensor]:\n",
    "        # split inputs and labels since they have to be of different lengths and need different padding methods\n",
    "        # first treat the audio inputs by simply returning torch tensors\n",
    "        input_features = [\n",
    "            {\"input_features\": feature[\"input_features\"][0]} for feature in features\n",
    "        ]\n",
    "        batch = self.processor.feature_extractor.pad(input_features, return_tensors=\"pt\")\n",
    "\n",
    "        # get the tokenized label sequences\n",
    "        label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n",
    "        # pad the labels to max length\n",
    "        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors=\"pt\")\n",
    "\n",
    "        # replace padding with -100 to ignore loss correctly\n",
    "        labels = labels_batch[\"input_ids\"].masked_fill(\n",
    "            labels_batch.attention_mask.ne(1), -100\n",
    "        )\n",
    "\n",
    "        # if bos token is appended in previous tokenization step,\n",
    "        # cut bos token here as it's append later anyways\n",
    "        if (labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():\n",
    "            labels = labels[:, 1:]\n",
    "\n",
    "        batch[\"labels\"] = labels\n",
    "\n",
    "        return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-22 09:57:50.784575: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-04-22 09:57:51.185398: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-04-22 09:57:52.418125: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import evaluate\n",
    "\n",
    "metric = evaluate.load(\"wer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers.models.whisper.english_normalizer import BasicTextNormalizer\n",
    "\n",
    "normalizer = BasicTextNormalizer()\n",
    "\n",
    "\n",
    "def compute_metrics(pred):\n",
    "    pred_ids = pred.predictions\n",
    "    label_ids = pred.label_ids\n",
    "\n",
    "    # replace -100 with the pad_token_id\n",
    "    label_ids[label_ids == -100] = processor.tokenizer.pad_token_id\n",
    "\n",
    "    # we do not want to group tokens when computing the metrics\n",
    "    pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)\n",
    "    label_str = processor.batch_decode(label_ids, skip_special_tokens=True)\n",
    "\n",
    "    # compute orthographic wer\n",
    "    wer_ortho = 100 * metric.compute(predictions=pred_str, references=label_str)\n",
    "\n",
    "    # compute normalised WER\n",
    "    pred_str_norm = [normalizer(pred) for pred in pred_str]\n",
    "    label_str_norm = [normalizer(label) for label in label_str]\n",
    "    # filtering step to only evaluate the samples that correspond to non-zero references:\n",
    "    pred_str_norm = [\n",
    "        pred_str_norm[i] for i in range(len(pred_str_norm)) if len(label_str_norm[i]) > 0\n",
    "    ]\n",
    "    label_str_norm = [\n",
    "        label_str_norm[i]\n",
    "        for i in range(len(label_str_norm))\n",
    "        if len(label_str_norm[i]) > 0\n",
    "    ]\n",
    "\n",
    "    wer = 100 * metric.compute(predictions=pred_str_norm, references=label_str_norm)\n",
    "\n",
    "    return {\"wer_ortho\": wer_ortho, \"wer\": wer}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import WhisperForConditionalGeneration\n",
    "\n",
    "model = WhisperForConditionalGeneration.from_pretrained(\"openai/whisper-small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "# disable cache during training since it's incompatible with gradient checkpointing\n",
    "model.config.use_cache = False\n",
    "\n",
    "# set language and task for generation and re-enable cache\n",
    "model.generate = partial(\n",
    "    model.generate, language=\"korean\", task=\"transcribe\", use_cache=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Seq2SeqTrainingArguments\n",
    "\n",
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=\"./whisper-small-ko\",  # name on the HF Hub\n",
    "    per_device_train_batch_size=16,\n",
    "    gradient_accumulation_steps=1,  # increase by 2x for every 2x decrease in batch size\n",
    "    learning_rate=1e-5,\n",
    "    lr_scheduler_type=\"constant_with_warmup\",\n",
    "    warmup_steps=50,\n",
    "    max_steps=1000,  # increase to 4000 if you have your own GPU or a Colab paid plan\n",
    "    gradient_checkpointing=True,\n",
    "    fp16=True,\n",
    "    fp16_full_eval=True,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    per_device_eval_batch_size=16,\n",
    "    predict_with_generate=True,\n",
    "    generation_max_length=225,\n",
    "    save_steps=500,\n",
    "    eval_steps=500,\n",
    "    logging_steps=25,\n",
    "    report_to=[\"tensorboard\"],\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=\"wer\",\n",
    "    greater_is_better=False,\n",
    "    push_to_hub=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shandy77/.local/lib/python3.10/site-packages/accelerate/accelerator.py:436: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
      "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import Seq2SeqTrainer\n",
    "\n",
    "trainer = Seq2SeqTrainer(\n",
    "    args=training_args,\n",
    "    model=model,\n",
    "    train_dataset=common_voice[\"train\"],\n",
    "    eval_dataset=common_voice[\"test\"],\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    "    tokenizer=processor,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shandy77/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1000' max='1000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1000/1000 39:08, Epoch 52/53]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Wer Ortho</th>\n",
       "      <th>Wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.664453</td>\n",
       "      <td>41.087613</td>\n",
       "      <td>40.483384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.682763</td>\n",
       "      <td>40.584089</td>\n",
       "      <td>40.080564</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
      "Non-default generation parameters: {'max_length': 448, 'suppress_tokens': [1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362], 'begin_suppress_tokens': [220, 50257]}\n",
      "/home/shandy77/.local/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
      "Non-default generation parameters: {'max_length': 448, 'suppress_tokens': [1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362], 'begin_suppress_tokens': [220, 50257]}\n",
      "There were missing keys in the checkpoint model loaded: ['proj_out.weight'].\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1000, training_loss=0.0006373535143720801, metrics={'train_runtime': 2352.0792, 'train_samples_per_second': 6.802, 'train_steps_per_second': 0.425, 'total_flos': 4.51232131940352e+18, 'train_loss': 0.0006373535143720801, 'epoch': 52.63})"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = {\n",
    "    \"dataset_tags\": \"mozilla-foundation/common_voice_13_0\",\n",
    "    \"dataset\": \"Common Voice 13\",  # a 'pretty' name for the training dataset\n",
    "    \"language\": \"ko\",\n",
    "    \"model_name\": \"Whisper Small Ko - Eveready\",  # a 'pretty' name for your model\n",
    "    \"finetuned_from\": \"openai/whisper-small\",\n",
    "    \"tasks\": \"automatic-speech-recognition\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
      "Non-default generation parameters: {'max_length': 448, 'suppress_tokens': [1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362], 'begin_suppress_tokens': [220, 50257]}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/Eveready/whisper-small-ko/commit/233d90eb7477c5e5f4043b4ebd87acea880e7b64', commit_message='End of training', commit_description='', oid='233d90eb7477c5e5f4043b4ebd87acea880e7b64', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub(**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}