# Kaldi language model

Examples
---
- [joshua 的LM製作教學](https://joshua.incubator.apache.org/6.0/large-lms.html)

Step 1: Count ngrams

#!/bin/bash
#NGRAM_COUNT=$SRILM_SRC/bin/i686-m64/ngram-count
args=""
stage=0
set -eo pipefail

. ./path.sh
. ./cmd.sh

NGRAM_COUNT=$KALDI_ROOT/tools/srilm/bin/i686-m64/ngram-count

source=data/train/corpus_formosa

. ./utils/parse_options.sh


IFS='/' read -ra arr <<< "$source"
name=${arr[-1]}
echo $name

if [ $stage -le 0 ]; then
  args=$args"-sort -order 5 -text $source -write counts/$name-counts.gz "
  echo $args | xargs --max-procs=4 -n 7 $NGRAM_COUNT
fi

Step 2: Make individual language models

#!/bin/bash
CMD_make_lm=$KALDI_ROOT/tools/srilm/bin/make-big-lm

source=
args=""
set -eo pipefail

. ./path.sh
. ./cmd.sh

. ./utils/parse_options.sh
PRUNE_THRESHOLD=1e-8
outdir=lms
mkdir -p ${outdir}

#-name gigalm `for k in counts/*.gz; do echo " \-read $k "; done` \
find . -iname "biglm.kncounts*" -exec rm -fr {} \;
find . -iname "gtcounts" -exec rm -fr {} \;
find . -iname "gigalm*" -exec rm -fr {} \;
$CMD_make_lm \
  -read $source \
  -lm ${outdir}/lm_${source/\//} \
  -max-per-file 100000000 \
  -order 3 \
  -kndiscount \
  -interpolate \
  -unk \
  -prune $PRUNE_THRESHOLD

Step 3: Mix models together
Mix model的流程是， 首先決定用什麼比例來合兩個model，這個步驟用3-1的繳本來計算他要用的參數， 才用3-2的腳本合成新LM
Step 3-1: Determine interpolation weights

#!/bin/bash

fold=0
args=""
stage=0
set -eo pipefail

. ../path.sh

rm -f */lm.ppl
NGRAM=$KALDI_ROOT/tools/srilm/bin/i686-m64/ngram
#DEV_TEXT=../data/train_fold0/corpus_train0
DEV_TEXT=../data/train_fold0/corpus_train${fold}

. ../utils/parse_options.sh

IFS='/' read -ra arr <<< "$DEV_TEXT"
name=${arr[-1]}

#dirs=( train${fold}-counts formosa-counts DAAIKID-counts )
dirs=( train${fold}-counts formosa-counts )
#dirs=( train${fold}_kid-counts formosa-counts DAAIKID-counts)
#dirs=( formosa-counts DAAIKID-counts)


for d in ${dirs[@]} ; do
  echo "$NGRAM -debug 2 -order 5 -unk -lm $d/lm.gz -ppl $DEV_TEXT > $d/lm.ppl ;"
  $NGRAM -debug 2 -order 5 -unk -lm $d/lm.gz -ppl $DEV_TEXT > $d/lm.ppl ;
done

compute-best-mix ${dirs[0]}/lm.ppl ${dirs[1]}/lm.ppl  > best-mix-fold${fold}.ppl

Step 3-2: Combine the models

#!/bin/bash

KALDI_ROOT=/home/jack/kaldi
export PATH=$PWD/../s5/utils/:$KALDI_ROOT/tools/srilm/bin/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/srilm/bin/i686-m64:$KALDI_ROOT/src/lmbin:$KALDI_ROOT/src/fstbin:$KALDI_ROOT/tools/openfst-1.6.7/src/bin:$PATH
fold=0
LAMBDAS=(0.0616934 0.938307)
. ../utils/parse_options.sh

NGRAM=$KALDI_ROOT/tools/srilm/bin/i686-m64/ngram

DIRS=( train${fold}-counts formosa-counts )
#LAMBDAS=(0.660804 0.339196) # apste the lambda weight by ourself
list0=(0.156418 0.843582)
list1=(0.144241 0.855759)
list3=(0.179678 0.820322)
list4=(0.182663 0.817337)
list5=(0.0616934 0.938307)
#LAMBDAS=(0.660804 0.339196) #for Shao's formosa + Esun
#parameters=(${list0} ${list1} ${list2} ${list3} ${list4})

echo $LAMBDAS

$NGRAM -order 3 -unk \
  -lm      ${DIRS[0]}/lm.gz    -lambda $LAMBDAS\
  -mix-lm  ${DIRS[1]}/lm.gz \
  -write-lm mixed_lm_fold${fold}.gz


echo "finished generating file: mixed_lm_fold${fold}.gz"

要注意，Mix 完model的時候要用diff指令檢查一下Mix後的結果有沒有跟Mix前的任何一個檔案一模一樣， 因為就算指令出錯了他也不會報錯

Step 3-3: Convert to Kaldi 的 lang_test
最後這邊是joshua沒有的， 這個腳本會生成Kaldi用到的G.fst

#!/bin/bash

rate=0.1
role=_groundtruth
. ./utils/parse_options.sh

#rate_suffix=_Rate${rate}

for fold in {0..4}; do
  utils/format_lm.sh data/lang lms/test_kid${fold}-counts/lm.gz data/local/dict/lexicon.txt data/lang_test${fold}${role};
done

echo "File generated to file data/lang_test${fold}${rate_suffix}"

腳本需要用的檔案:
---
上面所有腳本都會用到一個吃參數的腳本./utils/parse_options.sh 這個腳本可以從/$KALDI_ROOT/egs/wsj/s5/utils 裡面找到

###### tags: `kaldi`