From a4caebceb41c3ecf623ab0d96c527b57cf2a5b3d Mon Sep 17 00:00:00 2001
From: josh <jmeyer@mozilla.com>
Date: Wed, 12 Dec 2018 11:42:54 -0800
Subject: [PATCH] updates from transfer-learning

---
 .compute                 |  4 ++--
 .install                 |  2 +-
 evaluate.py              |  3 ++-
 filter_train_dev_test.py | 33 ++++++++++++++++-----------------
 4 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/.compute b/.compute
index 7b3ebc4c..803edb1e 100755
--- a/.compute
+++ b/.compute
@@ -35,9 +35,9 @@
 source ../tmp/venv/bin/activate
 
 # if num_layers_dropped=6, then we train from scratch (you dropped all the layers!)
-num_layers_dropped=5
+num_layers_dropped=6
 
-target_lang='de'
+target_lang='fr'
 source_model='en-v030'
 
 
diff --git a/.install b/.install
index 6b499993..ba67de54 100755
--- a/.install
+++ b/.install
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-transfer=1
+transfer=0
 
 ## SETUP VIRETUALENV ##
 virtualenv -p python3 ../tmp/venv
diff --git a/evaluate.py b/evaluate.py
index fd6b45ec..417e1f04 100755
--- a/evaluate.py
+++ b/evaluate.py
@@ -117,7 +117,8 @@ def evaluate(test_data, inference_graph, alphabet):
         sparse_labels = tf.cast(ctc_label_dense_to_sparse(labels_ph, label_lengths_ph, FLAGS.test_batch_size), tf.int32)
         loss = tf.nn.ctc_loss(labels=sparse_labels,
                               inputs=layers['raw_logits'],
-                              sequence_length=inputs['input_lengths'])
+                              sequence_length=inputs['input_lengths'],
+                              ignore_longer_outputs_than_inputs=True)
 
         # Create a saver using variables from the above newly created graph
         mapping = {v.op.name: v for v in tf.global_variables() if not v.op.name.startswith('previous_state_')}
diff --git a/filter_train_dev_test.py b/filter_train_dev_test.py
index 58c72800..8b169633 100644
--- a/filter_train_dev_test.py
+++ b/filter_train_dev_test.py
@@ -2,6 +2,7 @@ import pandas
 import sys
 import os
 import ntpath
+import glob
 
 '''
 USAGE:  $ python3 filter_cv1_dev_test.py DATA_DIR LOCALE CLIPS.TSV SAVE_TO_DIR
@@ -57,7 +58,6 @@ output_folder = sys.argv[4]
 
 print("Looking for clips.tsv here: ", clips_tsv)
 clips = pandas.read_csv(clips_tsv, sep='\t')
-
 print("### Total valid+invalid clips per lang ###")
 print(clips['locale'].value_counts())
 print("############################")
@@ -74,16 +74,6 @@ speaker_counts.columns= ['ID', 'counts']
 speaker_counts['cum_sum'] = speaker_counts.counts.cumsum()
 speaker_counts['cum_perc'] = 100 * speaker_counts.cum_sum / speaker_counts.counts.sum()
 
-### BIG LANGS ###
-# num_spks = len(speaker_counts.index)
-# train = ['train']*8
-# dev = ['dev']*1
-# test = ['test']*1
-# splits = train + dev + test
-# speaker_counts['new_bucket'] = pandas.Series((splits*num_spks)[:num_spks]).values
-##^ BIG LANGS ^##
-
-### LIL LANGS ###
 def bucket(row):
     if (row.cum_perc > 0.0) and( row.cum_perc < 80.0) :
         return "train"
@@ -91,12 +81,11 @@ def bucket(row):
         return "dev"
     elif (row.cum_perc > 90.0) and( row.cum_perc <= 100.0) :
         return "test"
+    
 speaker_counts.loc[:, 'new_bucket'] = speaker_counts.apply(bucket, axis = 1)
-##^ LIL LANGS ^##
-# speaker_counts.to_csv("speaker_count.csv", sep="\t")
-
-
 locale['new_bucket']=locale['ID'].map(speaker_counts.set_index('ID')['new_bucket'])
+### REASSIGN TEXT / DEV / TRAIN ###
+print(locale.head())
 
 
 locale['path'] = locale['path'].str.replace('/', '___')
@@ -112,13 +101,22 @@ train_paths = locale[locale['new_bucket'] == 'train'].loc[:, ['path']]
 ####                    ####
 
 # cv_LANG_valid.csv == wav_filename,wav_filesize,transcript
+### don't use splits on cluster quite yet ###
+all_files = glob.glob(os.path.join(data_dir, LOCALE, "*_valid_*.csv"))
+print((str(i) for i in all_files))
+df_from_each_file = (pandas.read_csv(f) for f in all_files)
+validated_clips   = pandas.concat(df_from_each_file, ignore_index=True)
+# validated_clips = pandas.read_csv('{}/{}/cv_{}_valid.csv'.format(data_dir, LOCALE, LOCALE))
+### don't use splits on cluster quite yet ###
+print(validated_clips.head())
 
-validated_clips = pandas.read_csv('{}/{}/cv_{}_valid.csv'.format(data_dir, LOCALE, LOCALE))
+# validated_clips = pandas.read_csv('{}/{}/cv_{}_valid.csv'.format(data_dir, LOCALE, LOCALE))
 validated_clips['path'] = validated_clips['wav_filename'].apply(ntpath.basename)
 validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'\xa0', ' ') # kyrgyz
 validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'\xad', ' ') # catalan
 validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'\\', ' ') # welsh
 
+print(validated_clips.head())
 
 
 ####              ####
@@ -129,10 +127,11 @@ validated_clips['transcript'] =  validated_clips['transcript'].str.replace(u'\\'
 dev_indices = validated_clips['path'].isin(dev_paths['path'])
 test_indices = validated_clips['path'].isin(test_paths['path'])
 train_indices = validated_clips['path'].isin(train_paths['path'])
+validated_clips['wav_filename'] =  data_dir + "/" + LOCALE + "/valid/" + validated_clips['path'].astype(str)
 validated_clips = validated_clips.drop(columns=['path'])
-validated_clips['wav_filename'] =  data_dir + "/" + LOCALE + "/" + validated_clips['wav_filename'].astype(str)
 
 
+print(validated_clips.head())
 
 
 ####              ####