From f534199534ebd3bbbd8e58d2e582c57e6fa1266f Mon Sep 17 00:00:00 2001
From: josh <jmeyer@mozilla.com>
Date: Thu, 13 Dec 2018 11:10:10 -0800
Subject: [PATCH] filtering and checking chars... somehow not syncing

---
 .compute                 |  2 +-
 data/alphabet.txt        | 93 +++++++++++++++-------------------------
 filter_train_dev_test.py |  3 ++
 util/check_characters.py |  2 +-
 4 files changed, 39 insertions(+), 61 deletions(-)

diff --git a/.compute b/.compute
index 6aabec89..1faab93a 100755
--- a/.compute
+++ b/.compute
@@ -37,7 +37,7 @@ source ../tmp/venv/bin/activate
 # train from scratch with num_layers_dropped=6
 num_layers_dropped=6
 
-target_lang='LANG'
+target_lang='cy'
 source_model='en-v030'
 
 
diff --git a/data/alphabet.txt b/data/alphabet.txt
index c98bc78f..ef74ced6 100644
--- a/data/alphabet.txt
+++ b/data/alphabet.txt
@@ -1,65 +1,40 @@
- 
--
-,
-!
-?
-.
-’
+–
 '
-\
-%
-3
-a
-A
-â
-ä
-b
-B
-c
-C
-d
-D
-e
-E
 ê
-f
-F
-g
-G
-h
-H
-i
-I
-î
-ï
-j
-J
-k
-K
-l
-L
-m
-M
-n
-N
-o
-O
-ô
 p
-P
-r
-R
-s
-S
-t
-T
-u
-U
-v
-w
-W
 ŵ
-y
-Y
+é
+a
+e
+g
+i
 ŷ
+m
+r
+u
+o
+b
+ä
+d
+ï
+s
+t
+y
+w
+’
+ö
+h
+c
+l
+â
+ô
+ 
+v
+k
+î
+n
+j
+á
+f
 z
+#
diff --git a/filter_train_dev_test.py b/filter_train_dev_test.py
index 8b169633..4f9f9c34 100644
--- a/filter_train_dev_test.py
+++ b/filter_train_dev_test.py
@@ -115,6 +115,9 @@ validated_clips['path'] = validated_clips['wav_filename'].apply(ntpath.basename)
 validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'\xa0', ' ') # kyrgyz
 validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'\xad', ' ') # catalan
 validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'\\', ' ') # welsh
+validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'„', ' ') # german
+validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'…', ' ') # german
+
 
 print(validated_clips.head())
 
diff --git a/util/check_characters.py b/util/check_characters.py
index 0e3f53e4..f407b96e 100644
--- a/util/check_characters.py
+++ b/util/check_characters.py
@@ -43,5 +43,5 @@ for inFile in (inFiles):
             csvFile.close()
 
 print("### The following unique characters were found in your transcripts: ###")
-print(list(allText))
+print(sorted(list(allText)))
 print("### All these characters should be in your data/alphabet.txt file ###")