E029 Sheffield
Python Modules
Data preparation for the E029 Sheffield CPC1 recipe. |
|
Recipe for training a Transformer ASR system with librispeech, from the SpeechBrain LibriSpeech/ASR recipe. |
|
|
Decoding methods for seq2seq autoregressive model. |
Configuration Files
config.yaml
1path:
2 root: ???
3 cpc1_train_data: ${path.root}/clarity_CPC1_data_train/
4 cpc1_test_data: ${path.root}/clarity_CPC1_data_test/
5 exp_folder: ${path.root}/e029
6
7cpc1_track: closed # "closed" or "open"
8dev_percent: 0.3 # amount of scenes for dev set
9
10MSBGEar: # hyperparameters for MSBG ear
11 src_pos: ff
12 sample_rate: 44100
13 equiv_0db_spl: 100
14 ahr: 20
15
16asr_config: transformer_cpc1.yaml
17
18hydra:
19 output_subdir: Null
20 run:
21 dir: .
22 job:
23 chdir: True
transformer_cpc1.yaml
1# ############################################################################
2# Model: E2E ASR with Transformer
3# Encoder: Transformer Encoder
4# Decoder: Transformer Decoder + (CTC/ATT joint) beamsearch + TransformerLM
5# Tokens: unigram
6# losses: CTC + KLdiv (Label Smoothing loss)
7# Training: Librispeech 960h
8# Authors: Jianyuan Zhong, Titouan Parcollet
9# ############################################################################
10# Seed needs to be set at top of yaml, before objects with parameters are made
11seed: 8886
12__set_seed: !apply:torch.manual_seed [!ref <seed>]
13output_folder: !ref ???
14wer_file: !ref <output_folder>/wer.txt
15save_folder: !ref <output_folder>/save
16train_log: !ref <output_folder>/train_log.txt
17
18# Language model (LM) pretraining
19# NB: To avoid mismatch, the speech recognizer must be trained with the same
20# tokenizer used for LM training. Here, we download everything from the
21# speechbrain HuggingFace repository. However, a local path pointing to a
22# directory containing the lm.ckpt and tokenizer.ckpt may also be specified
23# instead. E.g if you want to use your own LM / tokenizer.
24pretrained_lm_tokenizer_path: speechbrain/asr-transformer-transformerlm-librispeech
25
26# Data files
27data_folder: !ref ???
28train_splits: ["cpc1_train_binaural"]
29dev_splits: ["cpc1_dev_binaural"]
30test_splits: ["cpc1_train_left_dev", "cpc1_train_right_dev"]
31skip_prep: True
32train_csv: !ref <data_folder>/binaural_train_msbg.csv
33valid_csv: !ref <data_folder>/binaural_dev_msbg.csv
34test_csv:
35 - !ref <data_folder>/left_dev_msbg.csv
36 - !ref <data_folder>/right_dev_msbg.csv
37ckpt_interval_minutes: 30 # save checkpoint every N min
38
39# Training parameters
40# To make Transformers converge, the global bath size should be large enough.
41# The global batch size is computed as batch_size * n_gpus * gradient_accumulation.
42# Empirically, we found that this value should be >= 128.
43# Please, set your parameters accordingly.
44number_of_epochs: 120
45batch_size: 16
46ctc_weight: 0.3
47gradient_accumulation: 4
48gradient_clipping: 5.0
49loss_reduction: 'batchmean'
50sorting: random
51
52# stages related parameters
53stage_one_epochs: 90
54lr_adam: 1.0
55lr_sgd: 0.000025
56
57# Feature parameters
58sample_rate: 16000
59n_fft: 400
60n_mels: 80
61
62# Dataloader options
63train_dataloader_opts:
64 batch_size: !ref <batch_size>
65 shuffle: True
66
67valid_dataloader_opts:
68 batch_size: 1
69
70test_dataloader_opts:
71 batch_size: 1
72
73####################### Model parameters ###########################
74# Transformer
75d_model: 768
76nhead: 8
77num_encoder_layers: 12
78num_decoder_layers: 6
79d_ffn: 3072
80transformer_dropout: 0.0
81activation: !name:torch.nn.GELU
82output_neurons: 5000
83vocab_size: 5000
84
85# Outputs
86blank_index: 0
87label_smoothing: 0.1
88pad_index: 0
89bos_index: 1
90eos_index: 2
91unk_index: 0
92
93# Decoding parameters
94min_decode_ratio: 0.0
95max_decode_ratio: 1.0
96valid_search_interval: 10
97valid_beam_size: 10
98test_beam_size: 66
99lm_weight: 0.00
100ctc_weight_decode: 0.40
101topk: 10
102n_ensembles: 6
103temperature: 1
104
105############################## models ################################
106
107CNN: !new:speechbrain.lobes.models.convolution.ConvolutionFrontEnd
108 input_shape: (8, 10, 80)
109 num_blocks: 3
110 num_layers_per_block: 1
111 out_channels: (128, 256, 512)
112 kernel_sizes: (3, 3, 1)
113 strides: (2, 2, 1)
114 residuals: (False, False, False)
115
116Transformer: !new:speechbrain.lobes.models.transformer.TransformerASR.TransformerASR # yamllint disable-line rule:line-length
117 input_size: 10240
118 tgt_vocab: !ref <output_neurons>
119 d_model: !ref <d_model>
120 nhead: !ref <nhead>
121 num_encoder_layers: !ref <num_encoder_layers>
122 num_decoder_layers: !ref <num_decoder_layers>
123 d_ffn: !ref <d_ffn>
124 dropout: !ref <transformer_dropout>
125 activation: !ref <activation>
126 normalize_before: False
127
128# This is the TransformerLM that is used according to the Huggingface repository
129# Visit the HuggingFace model corresponding to the pretrained_lm_tokenizer_path
130# For more details about the model!
131# NB: It has to match the pre-trained TransformerLM!!
132lm_model: !new:speechbrain.lobes.models.transformer.TransformerLM.TransformerLM # yamllint disable-line rule:line-length
133 vocab: !ref <output_neurons>
134 d_model: 768
135 nhead: 12
136 num_encoder_layers: 12
137 num_decoder_layers: 0
138 d_ffn: 3072
139 dropout: 0.0
140 activation: !name:torch.nn.GELU
141 normalize_before: False
142
143tokenizer: !new:sentencepiece.SentencePieceProcessor
144
145ctc_lin: !new:speechbrain.nnet.linear.Linear
146 input_size: !ref <d_model>
147 n_neurons: !ref <output_neurons>
148
149seq_lin: !new:speechbrain.nnet.linear.Linear
150 input_size: !ref <d_model>
151 n_neurons: !ref <output_neurons>
152
153modules:
154 CNN: !ref <CNN>
155 Transformer: !ref <Transformer>
156 seq_lin: !ref <seq_lin>
157 ctc_lin: !ref <ctc_lin>
158
159model: !new:torch.nn.ModuleList
160 - [!ref <CNN>, !ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>]
161
162# define two optimizers here for two-stage training
163Adam: !name:torch.optim.Adam
164 lr: 0
165 betas: (0.9, 0.98)
166 eps: 0.000000001
167
168SGD: !name:torch.optim.SGD
169 lr: !ref <lr_sgd>
170 momentum: 0.99
171 nesterov: True
172
173valid_search: !new:speechbrain.decoders.S2STransformerBeamSearch
174 modules: [!ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>]
175 bos_index: !ref <bos_index>
176 eos_index: !ref <eos_index>
177 blank_index: !ref <blank_index>
178 min_decode_ratio: !ref <min_decode_ratio>
179 max_decode_ratio: !ref <max_decode_ratio>
180 beam_size: !ref <valid_beam_size>
181 ctc_weight: !ref <ctc_weight_decode>
182 using_eos_threshold: False
183 length_normalization: False
184
185
186test_search: !new:speechbrain.decoders.S2STransformerBeamSearch
187 modules: [!ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>]
188 bos_index: !ref <bos_index>
189 eos_index: !ref <eos_index>
190 blank_index: !ref <blank_index>
191 min_decode_ratio: !ref <min_decode_ratio>
192 max_decode_ratio: !ref <max_decode_ratio>
193 beam_size: !ref <test_beam_size>
194 ctc_weight: !ref <ctc_weight_decode>
195 lm_weight: !ref <lm_weight>
196 lm_modules: !ref <lm_model>
197 temperature: 1
198 temperature_lm: 1
199 using_eos_threshold: False
200 length_normalization: True
201
202log_softmax: !new:torch.nn.LogSoftmax
203 dim: -1
204
205ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
206 blank_index: !ref <blank_index>
207 reduction: !ref <loss_reduction>
208
209seq_cost: !name:speechbrain.nnet.losses.kldiv_loss
210 label_smoothing: !ref <label_smoothing>
211 reduction: !ref <loss_reduction>
212
213noam_annealing: !new:speechbrain.nnet.schedulers.NoamScheduler
214 lr_initial: !ref <lr_adam>
215 n_warmup_steps: 25000
216 model_size: !ref <d_model>
217
218checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
219 checkpoints_dir: !ref <save_folder>
220 recoverables:
221 model: !ref <model>
222 noam_scheduler: !ref <noam_annealing>
223 normalizer: !ref <normalize>
224 counter: !ref <epoch_counter>
225
226epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
227 limit: !ref <number_of_epochs>
228
229normalize: !new:speechbrain.processing.features.InputNormalization
230 norm_type: global
231 update_until_epoch: 4
232
233augmentation: !new:speechbrain.lobes.augment.SpecAugment
234 time_warp: True
235 time_warp_window: 5
236 time_warp_mode: bicubic
237 freq_mask: True
238 n_freq_mask: 2
239 time_mask: True
240 n_time_mask: 2
241 replace_with_zero: False
242 freq_mask_width: 30
243 time_mask_width: 40
244
245speed_perturb: !new:speechbrain.processing.speech_augmentation.SpeedPerturb
246 orig_freq: !ref <sample_rate>
247 speeds: [95, 100, 105]
248
249compute_features: !new:speechbrain.lobes.features.Fbank
250 sample_rate: !ref <sample_rate>
251 n_fft: !ref <n_fft>
252 n_mels: !ref <n_mels>
253
254train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
255 save_file: !ref <train_log>
256
257error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
258acc_computer: !name:speechbrain.utils.Accuracy.AccuracyStats
259
260# The pretrainer allows a mapping between pretrained files and instances that
261# are declared in the yaml. E.g here, we will download the file lm.ckpt
262# and it will be loaded into "lm" which is pointing to the <lm_model> defined
263# before.
264pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
265 collect_in: !ref <save_folder>
266 loadables:
267 lm: !ref <lm_model>
268 tokenizer: !ref <tokenizer>
269 paths:
270 lm: !ref <pretrained_lm_tokenizer_path>/lm.ckpt
271 tokenizer: !ref <pretrained_lm_tokenizer_path>/tokenizer.ckpt