diff --git a/.compute b/.compute index 0a173543..484f873a 100644 --- a/.compute +++ b/.compute @@ -14,20 +14,25 @@ pip install $(python3 util/taskcluster.py --decoder) mkdir -p ../keep/summaries LANG="sl" -cv="$SHARED_DIR/data/mozilla/CommonVoice/v2.0-alpha2.0/$LANG" +CV="${SHARED_DIR}/data/mozilla/CommonVoice/v2.0-alpha2.0/${LANG}" + +# the *.csv on cluster have old paths +cp ${CV}/*.csv . +sed -Ei 's/snakepit/data\/ro/g' cv_${LANG}_valid_*.csv + python -u DeepSpeech.py \ - --train_files '$CV/cv_${LANG}_valid_train.csv' \ - --dev_files '$CV/cv_${LANG}_valid_dev.csv' \ - --test_files '$CV/cv_${LANG}_valid_test.csv' \ + --train_files "cv_${LANG}_valid_train.csv" \ + --dev_files "cv_${LANG}_valid_dev.csv" \ + --test_files "cv_${LANG}_valid_test.csv" \ --train_batch_size 24 \ --dev_batch_size 48 \ --test_batch_size 48 \ --noearly_stop \ --n_hidden 2048 \ --learning_rate 0.0001 \ - --dropout_rate 0.15 \ - --epoch 1 \ + --dropout_rate 0.2 \ + --epoch 30 \ --display_step 0 \ --validation_step 1 \ --checkpoint_dir "../keep" \ diff --git a/util/create_alpha_lm_trie.sh b/util/create_alpha_lm_trie.sh new file mode 100644 index 00000000..b3d51620 --- /dev/null +++ b/util/create_alpha_lm_trie.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +LANG=$1 +TEXT=$2 + +echo "$0: Looking for CSV transcripts at cv_${LANG}_valid_{train/dev/test}.csv" +echo "$0: Looking for text training corpus at ${TEXT}" + +# kenlm Dependencies +apt-get install -y build-essential cmake libboost-all-dev zlib1g-dev libbz2-dev liblzma-dev libeigen3-dev + +# Install Kenlm # + +wget -O - https://kheafield.com/code/kenlm.tar.gz | tar xz +mkdir kenlm/build +cd kenlm/build +cmake .. +make -j `nproc` +cd ../.. + +################# +### CREATE LM ### +################# + +# Make alphabet.txt # + +python3 util/check_characters.py \ + -csv "cv_${LANG}_valid_train.csv","cv_${LANG}_valid_train.csv","cv_${LANG}_valid_train.csv" \ + -alpha \ + | data/alphabet.txt + +# Make lm.arpa # + +kenlm/build/bin/lmplz \ + --order 2 \ + --text ${TEXT} \ + --arpa /tmp/lm.arpa + +# Make lm.binary # + +kenlm/build/bin/build_binary \ + -a 255 \ + -q 8 trie \ + /tmp/lm.arpa \ + data/lm/lm.binary + +# Make trie # + +native_client/generate_trie \ + data/alphabet.txt \ + data/lm/lm.binary \ + data/lm/trie + +rm /tmp/lm.arpa + +