E032 Sheffield
Python Modules
Data preparation for the E032 Sheffield CPC1 recipe. |
|
Recipe for training a Transformer ASR system with librispeech, from the SpeechBrain LibriSpeech/ASR recipe. |
|
Decoding methods for seq2seq autoregressive model. |
Configuration Files
config.yaml
1path:
2 root: ???
3 cpc1_train_data: ${path.root}/clarity_CPC1_data_train/
4 cpc1_test_data: ${path.root}/clarity_CPC1_data_test/
5 exp_folder: ${path.root}/e032
6
7cpc1_track: closed # "closed" or "open"
8dev_percent: 0.3 # amount of scenes for dev set
9
10MSBGEar: # hyperparameters for MSBG ear
11 src_pos: ff
12 sample_rate: 44100
13 equiv_0db_spl: 100
14 ahr: 20
15
16asr_config: transformer_cpc1.yaml
17
18hydra:
19 output_subdir: Null
20 run:
21 dir: .
22 job:
23 chdir: True
transformer_cpc1.yaml
1# ############################################################################
2# Model: E2E ASR with Transformer
3# Encoder: Transformer Encoder
4# Decoder: Transformer Decoder + (CTC/ATT joint) beamsearch + TransformerLM
5# Tokens: unigram
6# losses: CTC + KLdiv (Label Smoothing loss)
7# Training: Librispeech 960h
8# Authors: Jianyuan Zhong, Titouan Parcollet
9# ############################################################################
10# Seed needs to be set at top of yaml, before objects with parameters are made
11seed: 8886
12__set_seed: !apply:torch.manual_seed [!ref <seed>]
13output_folder: !ref ???
14wer_file: !ref <output_folder>/wer.txt
15save_folder: !ref <output_folder>/save
16train_log: !ref <output_folder>/train_log.txt
17
18# Language model (LM) pretraining
19# NB: To avoid mismatch, the speech recognizer must be trained with the same
20# tokenizer used for LM training. Here, we download everything from the
21# speechbrain HuggingFace repository. However, a local path pointing to a
22# directory containing the lm.ckpt and tokenizer.ckpt may also be specified
23# instead. E.g if you want to use your own LM / tokenizer.
24pretrained_lm_tokenizer_path: speechbrain/asr-transformer-transformerlm-librispeech
25
26# Data files
27data_folder: !ref ???
28train_splits: ["cpc1_train_binaural"]
29dev_splits: ["cpc1_dev_binaural"]
30test_splits: ["cpc1_train_left_dev", "cpc1_train_right_dev"]
31skip_prep: True
32train_csv: !ref <data_folder>/binaural_train_msbg.csv
33valid_csv: !ref <data_folder>/binaural_dev_msbg.csv
34test_csv:
35 - !ref <data_folder>/left_dev_msbg.csv
36 - !ref <data_folder>/right_dev_msbg.csv
37ckpt_interval_minutes: 30 # save checkpoint every N min
38
39# Training parameters
40# To make Transformers converge, the global bath size should be large enough.
41# The global batch size is computed as batch_size * n_gpus * gradient_accumulation.
42# Empirically, we found that this value should be >= 128.
43# Please, set your parameters accordingly.
44number_of_epochs: 120
45batch_size: 16
46ctc_weight: 0.3
47gradient_accumulation: 4
48gradient_clipping: 5.0
49loss_reduction: 'batchmean'
50sorting: random
51
52# stages related parameters
53stage_one_epochs: 90
54lr_adam: 1.0
55lr_sgd: 0.000025
56
57# Feature parameters
58sample_rate: 16000
59n_fft: 400
60n_mels: 80
61
62# Dataloader options
63train_dataloader_opts:
64 batch_size: !ref <batch_size>
65 shuffle: True
66
67valid_dataloader_opts:
68 batch_size: 1
69
70test_dataloader_opts:
71 batch_size: 1
72
73####################### Model parameters ###########################
74# Transformer
75d_model: 768
76nhead: 8
77num_encoder_layers: 12
78num_decoder_layers: 6
79d_ffn: 3072
80transformer_dropout: 0.0
81activation: !name:torch.nn.GELU
82output_neurons: 5000
83vocab_size: 5000
84
85# Outputs
86blank_index: 0
87label_smoothing: 0.1
88pad_index: 0
89bos_index: 1
90eos_index: 2
91unk_index: 0
92
93# Decoding parameters
94min_decode_ratio: 0.0
95max_decode_ratio: 1.0
96valid_search_interval: 10
97valid_beam_size: 10
98test_beam_size: 66
99lm_weight: 0.00
100ctc_weight_decode: 0.40
101
102############################## models ################################
103
104CNN: !new:speechbrain.lobes.models.convolution.ConvolutionFrontEnd
105 input_shape: (8, 10, 80)
106 num_blocks: 3
107 num_layers_per_block: 1
108 out_channels: (128, 256, 512)
109 kernel_sizes: (3, 3, 1)
110 strides: (2, 2, 1)
111 residuals: (False, False, False)
112
113Transformer: !new:speechbrain.lobes.models.transformer.TransformerASR.TransformerASR # yamllint disable-line rule:line-length
114 input_size: 10240
115 tgt_vocab: !ref <output_neurons>
116 d_model: !ref <d_model>
117 nhead: !ref <nhead>
118 num_encoder_layers: !ref <num_encoder_layers>
119 num_decoder_layers: !ref <num_decoder_layers>
120 d_ffn: !ref <d_ffn>
121 dropout: !ref <transformer_dropout>
122 activation: !ref <activation>
123 normalize_before: False
124
125# This is the TransformerLM that is used according to the Huggingface repository
126# Visit the HuggingFace model corresponding to the pretrained_lm_tokenizer_path
127# For more details about the model!
128# NB: It has to match the pre-trained TransformerLM!!
129lm_model: !new:speechbrain.lobes.models.transformer.TransformerLM.TransformerLM # yamllint disable-line rule:line-length
130 vocab: !ref <output_neurons>
131 d_model: 768
132 nhead: 12
133 num_encoder_layers: 12
134 num_decoder_layers: 0
135 d_ffn: 3072
136 dropout: 0.0
137 activation: !name:torch.nn.GELU
138 normalize_before: False
139
140tokenizer: !new:sentencepiece.SentencePieceProcessor
141
142ctc_lin: !new:speechbrain.nnet.linear.Linear
143 input_size: !ref <d_model>
144 n_neurons: !ref <output_neurons>
145
146seq_lin: !new:speechbrain.nnet.linear.Linear
147 input_size: !ref <d_model>
148 n_neurons: !ref <output_neurons>
149
150modules:
151 CNN: !ref <CNN>
152 Transformer: !ref <Transformer>
153 seq_lin: !ref <seq_lin>
154 ctc_lin: !ref <ctc_lin>
155
156model: !new:torch.nn.ModuleList
157 - [!ref <CNN>, !ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>]
158
159# define two optimizers here for two-stage training
160Adam: !name:torch.optim.Adam
161 lr: 0
162 betas: (0.9, 0.98)
163 eps: 0.000000001
164
165SGD: !name:torch.optim.SGD
166 lr: !ref <lr_sgd>
167 momentum: 0.99
168 nesterov: True
169
170valid_search: !new:speechbrain.decoders.S2STransformerBeamSearch
171 modules: [!ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>]
172 bos_index: !ref <bos_index>
173 eos_index: !ref <eos_index>
174 blank_index: !ref <blank_index>
175 min_decode_ratio: !ref <min_decode_ratio>
176 max_decode_ratio: !ref <max_decode_ratio>
177 beam_size: !ref <valid_beam_size>
178 ctc_weight: !ref <ctc_weight_decode>
179 using_eos_threshold: False
180 length_normalization: False
181
182
183test_search: !new:speechbrain.decoders.S2STransformerBeamSearch
184 modules: [!ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>]
185 bos_index: !ref <bos_index>
186 eos_index: !ref <eos_index>
187 blank_index: !ref <blank_index>
188 min_decode_ratio: !ref <min_decode_ratio>
189 max_decode_ratio: !ref <max_decode_ratio>
190 beam_size: !ref <test_beam_size>
191 ctc_weight: !ref <ctc_weight_decode>
192 lm_weight: !ref <lm_weight>
193 lm_modules: !ref <lm_model>
194 temperature: 1
195 temperature_lm: 1
196 using_eos_threshold: False
197 length_normalization: True
198
199log_softmax: !new:torch.nn.LogSoftmax
200 dim: -1
201
202ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
203 blank_index: !ref <blank_index>
204 reduction: !ref <loss_reduction>
205
206seq_cost: !name:speechbrain.nnet.losses.kldiv_loss
207 label_smoothing: !ref <label_smoothing>
208 reduction: !ref <loss_reduction>
209
210noam_annealing: !new:speechbrain.nnet.schedulers.NoamScheduler
211 lr_initial: !ref <lr_adam>
212 n_warmup_steps: 25000
213 model_size: !ref <d_model>
214
215checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
216 checkpoints_dir: !ref <save_folder>
217 recoverables:
218 model: !ref <model>
219 noam_scheduler: !ref <noam_annealing>
220 normalizer: !ref <normalize>
221 counter: !ref <epoch_counter>
222
223epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
224 limit: !ref <number_of_epochs>
225
226normalize: !new:speechbrain.processing.features.InputNormalization
227 norm_type: global
228 update_until_epoch: 4
229
230augmentation: !new:speechbrain.lobes.augment.SpecAugment
231 time_warp: True
232 time_warp_window: 5
233 time_warp_mode: bicubic
234 freq_mask: True
235 n_freq_mask: 2
236 time_mask: True
237 n_time_mask: 2
238 replace_with_zero: False
239 freq_mask_width: 30
240 time_mask_width: 40
241
242speed_perturb: !new:speechbrain.processing.speech_augmentation.SpeedPerturb
243 orig_freq: !ref <sample_rate>
244 speeds: [95, 100, 105]
245
246compute_features: !new:speechbrain.lobes.features.Fbank
247 sample_rate: !ref <sample_rate>
248 n_fft: !ref <n_fft>
249 n_mels: !ref <n_mels>
250
251train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
252 save_file: !ref <train_log>
253
254error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
255acc_computer: !name:speechbrain.utils.Accuracy.AccuracyStats
256
257# The pretrainer allows a mapping between pretrained files and instances that
258# are declared in the yaml. E.g here, we will download the file lm.ckpt
259# and it will be loaded into "lm" which is pointing to the <lm_model> defined
260# before.
261pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
262 collect_in: !ref <save_folder>
263 loadables:
264 lm: !ref <lm_model>
265 tokenizer: !ref <tokenizer>
266 paths:
267 lm: !ref <pretrained_lm_tokenizer_path>/lm.ckpt
268 tokenizer: !ref <pretrained_lm_tokenizer_path>/tokenizer.ckpt