pszemraj commited on
Commit
795adfb
1 Parent(s): c61dbdb
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ checkpoint-*/
config.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "_name_or_path": "pszemraj/long-t5-tglobal-base-16384-booksum-V6-partial",
3
  "architectures": [
4
  "LongT5ForConditionalGeneration"
5
  ],
@@ -36,7 +36,7 @@
36
  "relative_attention_num_buckets": 32,
37
  "repetition_penalty": 3.5,
38
  "tie_word_embeddings": false,
39
- "torch_dtype": "bfloat16",
40
  "transformers_version": "4.20.1",
41
  "use_cache": false,
42
  "vocab_size": 32128
 
1
  {
2
+ "_name_or_path": "pszemraj/long-t5-tglobal-base-16384-booksum-V7.9",
3
  "architectures": [
4
  "LongT5ForConditionalGeneration"
5
  ],
 
36
  "relative_attention_num_buckets": 32,
37
  "repetition_penalty": 3.5,
38
  "tie_word_embeddings": false,
39
+ "torch_dtype": "float32",
40
  "transformers_version": "4.20.1",
41
  "use_cache": false,
42
  "vocab_size": 32128
latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step330
long-t5-tglobal-base-16384-booksum-V7.9-ft1-booksum_training_metadata.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"output_dir": "/content/drive/MyDrive/Programming/hf-trainer/long-t5-tglobal-base-16384-booksum-V7.9-ft1-booksum", "overwrite_output_dir": true, "do_train": false, "do_eval": false, "do_predict": false, "evaluation_strategy": "no", "prediction_loss_only": false, "per_device_train_batch_size": 1, "per_device_eval_batch_size": 1, "per_gpu_train_batch_size": "None", "per_gpu_eval_batch_size": "None", "gradient_accumulation_steps": 64, "eval_accumulation_steps": "None", "eval_delay": 0, "learning_rate": 0.001, "weight_decay": 0.05, "adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "max_grad_norm": 0.5, "num_train_epochs": 2, "max_steps": -1, "lr_scheduler_type": "cosine", "warmup_ratio": 0.01, "warmup_steps": 0, "log_level": -1, "log_level_replica": -1, "log_on_each_node": true, "logging_dir": "/content/drive/MyDrive/Programming/hf-trainer/long-t5-tglobal-base-16384-booksum-V7.9-ft1-booksum/logs", "logging_strategy": "steps", "logging_first_step": false, "logging_steps": 2, "logging_nan_inf_filter": true, "save_strategy": "epoch", "save_steps": 500, "save_total_limit": 1, "save_on_each_node": false, "no_cuda": false, "seed": 42, "data_seed": "None", "jit_mode_eval": false, "use_ipex": false, "bf16": false, "fp16": true, "fp16_opt_level": "O1", "half_precision_backend": "cuda_amp", "bf16_full_eval": false, "fp16_full_eval": false, "tf32": "None", "local_rank": 0, "xpu_backend": "None", "tpu_num_cores": "None", "tpu_metrics_debug": false, "debug": "[]", "dataloader_drop_last": false, "eval_steps": "None", "dataloader_num_workers": 0, "past_index": -1, "run_name": "/content/drive/MyDrive/Programming/hf-trainer/long-t5-tglobal-base-16384-booksum-V7.9-ft1-booksum", "disable_tqdm": false, "remove_unused_columns": true, "label_names": "None", "load_best_model_at_end": false, "metric_for_best_model": "None", "greater_is_better": "None", "ignore_data_skip": false, "sharded_ddp": "[]", "fsdp": "[]", "fsdp_min_num_params": 0, "deepspeed": "/content/ds_config_zero2.json", "label_smoothing_factor": 0.0, "optim": "adamw_hf", "adafactor": false, "group_by_length": false, "length_column_name": "length", "report_to": "['tensorboard']", "ddp_find_unused_parameters": "None", "ddp_bucket_cap_mb": "None", "dataloader_pin_memory": true, "skip_memory_metrics": true, "use_legacy_prediction_loop": false, "push_to_hub": true, "resume_from_checkpoint": "None", "hub_model_id": "long-t5-tglobal-base-16384-booksum-V7.9-ft1-booksum", "hub_strategy": "end", "hub_token": "<HUB_TOKEN>", "hub_private_repo": true, "gradient_checkpointing": true, "include_inputs_for_metrics": false, "fp16_backend": "auto", "push_to_hub_model_id": "None", "push_to_hub_organization": "None", "push_to_hub_token": "<PUSH_TO_HUB_TOKEN>", "_n_gpu": 1, "mp_parameters": "", "auto_find_batch_size": false, "full_determinism": false, "torchdynamo": "None", "ray_scope": "last", "sortish_sampler": false, "predict_with_generate": false, "generation_max_length": "None", "generation_num_beams": "None", "train_batch_size": 1, "eval_batch_size": 1, "configs_src": "long-t5-tglobal-base-16384-booksum-V7.9-ft1-booksum"}
pytorch_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4adb405b88e1acd599988a723bb03f622f2bc5b7b441d9a480ed5b5ed75fa190
3
  size 990388907
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b05f4b28e354b9cc1c758956764bfd54d590226a1dfbe604856ded1dbafd148e
3
  size 990388907
tokenizer_config.json CHANGED
@@ -103,7 +103,7 @@
103
  ],
104
  "eos_token": "</s>",
105
  "extra_ids": 100,
106
- "name_or_path": "pszemraj/long-t5-tglobal-base-16384-booksum-V6-partial",
107
  "pad_token": "<pad>",
108
  "special_tokens_map_file": null,
109
  "tokenizer_class": "T5Tokenizer",
 
103
  ],
104
  "eos_token": "</s>",
105
  "extra_ids": 100,
106
+ "name_or_path": "pszemraj/long-t5-tglobal-base-16384-booksum-V7.9",
107
  "pad_token": "<pad>",
108
  "special_tokens_map_file": null,
109
  "tokenizer_class": "T5Tokenizer",
trainer_state.json CHANGED
@@ -1,349 +1,1015 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 1.991123701605288,
5
- "global_step": 164,
6
  "is_hyper_param_search": false,
7
  "is_local_process_zero": true,
8
  "is_world_process_zero": true,
9
  "log_history": [
 
 
 
 
 
 
 
 
 
 
 
 
10
  {
11
  "epoch": 0.04,
12
- "learning_rate": 0.00030000000000000003,
13
- "loss": 2.5934,
14
- "step": 3
 
 
 
 
 
 
 
 
 
 
 
 
15
  },
16
  {
17
  "epoch": 0.07,
18
- "learning_rate": 0.0003998458072481446,
19
- "loss": 2.5887,
20
- "step": 6
 
 
 
 
 
 
 
 
 
 
 
 
21
  },
22
  {
23
  "epoch": 0.11,
24
- "learning_rate": 0.0003990369453344394,
25
- "loss": 2.5383,
26
- "step": 9
 
 
 
 
 
 
 
 
 
 
 
 
27
  },
28
  {
29
  "epoch": 0.15,
30
- "learning_rate": 0.00039753766811902755,
31
- "loss": 2.5382,
32
- "step": 12
 
 
 
 
 
 
 
 
 
 
 
 
33
  },
34
  {
35
  "epoch": 0.18,
36
- "learning_rate": 0.0003953531762641745,
37
- "loss": 2.5684,
38
- "step": 15
 
 
 
 
 
 
 
 
 
 
 
 
39
  },
40
  {
41
  "epoch": 0.22,
42
- "learning_rate": 0.00039249104729072946,
43
- "loss": 2.5694,
44
- "step": 18
 
 
 
 
 
 
 
 
 
 
 
 
45
  },
46
  {
47
  "epoch": 0.25,
48
- "learning_rate": 0.00038896120929337566,
49
- "loss": 2.5685,
50
- "step": 21
 
 
 
 
 
 
 
 
 
 
 
 
51
  },
52
  {
53
  "epoch": 0.29,
54
- "learning_rate": 0.0003847759065022574,
55
- "loss": 2.5249,
56
- "step": 24
 
 
 
 
 
 
 
 
 
 
 
 
57
  },
58
  {
59
  "epoch": 0.33,
60
- "learning_rate": 0.00037994965681044433,
61
- "loss": 2.5835,
62
- "step": 27
 
 
 
 
 
 
 
 
 
 
 
 
63
  },
64
  {
65
  "epoch": 0.36,
66
- "learning_rate": 0.00037449920141455944,
67
- "loss": 2.5326,
68
- "step": 30
 
 
 
 
 
 
 
 
 
 
 
 
69
  },
70
  {
71
  "epoch": 0.4,
72
- "learning_rate": 0.00036844344674325733,
73
- "loss": 2.564,
74
- "step": 33
 
 
 
 
 
 
 
 
 
 
 
 
75
  },
76
  {
77
  "epoch": 0.44,
78
- "learning_rate": 0.0003618033988749895,
79
- "loss": 2.5717,
80
- "step": 36
 
 
 
 
 
 
 
 
 
 
 
 
81
  },
82
  {
83
  "epoch": 0.47,
84
- "learning_rate": 0.0003546020906725474,
85
- "loss": 2.5149,
86
- "step": 39
 
 
 
 
 
 
 
 
 
 
 
 
87
  },
88
  {
89
  "epoch": 0.51,
90
- "learning_rate": 0.0003468645018871371,
91
- "loss": 2.5695,
92
- "step": 42
 
 
 
 
 
 
 
 
 
 
 
 
93
  },
94
  {
95
  "epoch": 0.54,
96
- "learning_rate": 0.0003386174725091272,
97
- "loss": 2.5374,
98
- "step": 45
 
 
 
 
 
 
 
 
 
 
 
 
99
  },
100
  {
101
  "epoch": 0.58,
102
- "learning_rate": 0.0003298896096660367,
103
- "loss": 2.5413,
104
- "step": 48
 
 
 
 
 
 
 
 
 
 
 
 
105
  },
106
  {
107
  "epoch": 0.62,
108
- "learning_rate": 0.0003207111883907143,
109
- "loss": 2.509,
110
- "step": 51
 
 
 
 
 
 
 
 
 
 
 
 
111
  },
112
  {
113
  "epoch": 0.65,
114
- "learning_rate": 0.00031111404660392046,
115
- "loss": 2.5628,
116
- "step": 54
 
 
 
 
 
 
 
 
 
 
 
 
117
  },
118
  {
119
  "epoch": 0.69,
120
- "learning_rate": 0.00030113147467559695,
121
- "loss": 2.5069,
122
- "step": 57
 
 
 
 
 
 
 
 
 
 
 
 
123
  },
124
  {
125
  "epoch": 0.73,
126
- "learning_rate": 0.00029079809994790937,
127
- "loss": 2.5331,
128
- "step": 60
 
 
 
 
 
 
 
 
 
 
 
 
129
  },
130
  {
131
  "epoch": 0.76,
132
- "learning_rate": 0.0002801497666206282,
133
- "loss": 2.5527,
134
- "step": 63
 
 
 
 
 
 
 
 
 
 
 
 
135
  },
136
  {
137
  "epoch": 0.8,
138
- "learning_rate": 0.0002692234114154986,
139
- "loss": 2.6179,
140
- "step": 66
 
 
 
 
 
 
 
 
 
 
 
 
141
  },
142
  {
143
  "epoch": 0.83,
144
- "learning_rate": 0.00025805693545089247,
145
- "loss": 2.5411,
146
- "step": 69
 
 
 
 
 
 
 
 
 
 
 
 
147
  },
148
  {
149
  "epoch": 0.87,
150
- "learning_rate": 0.00024668907277118114,
151
- "loss": 2.5583,
152
- "step": 72
 
 
 
 
 
 
 
 
 
 
 
 
153
  },
154
  {
155
  "epoch": 0.91,
156
- "learning_rate": 0.00023515925598687094,
157
- "loss": 2.534,
158
- "step": 75
 
 
 
 
 
 
 
 
 
 
 
 
159
  },
160
  {
161
  "epoch": 0.94,
162
- "learning_rate": 0.00022350747949156756,
163
- "loss": 2.5433,
164
- "step": 78
 
 
 
 
 
 
 
 
 
 
 
 
165
  },
166
  {
167
  "epoch": 0.98,
168
- "learning_rate": 0.0002117741607302378,
169
- "loss": 2.5487,
170
- "step": 81
 
 
 
 
 
 
 
 
 
 
 
 
171
  },
172
  {
173
  "epoch": 1.02,
174
- "learning_rate": 0.0002,
175
- "loss": 3.1324,
176
- "step": 84
 
 
 
 
 
 
177
  },
178
  {
179
- "epoch": 1.06,
180
- "learning_rate": 0.00018822583926976218,
181
- "loss": 2.4641,
182
- "step": 87
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  },
184
  {
185
  "epoch": 1.1,
186
- "learning_rate": 0.00017649252050843252,
187
- "loss": 2.4806,
188
- "step": 90
 
 
 
 
 
 
189
  },
190
  {
191
  "epoch": 1.13,
192
- "learning_rate": 0.0001648407440131291,
193
- "loss": 2.4693,
194
- "step": 93
195
  },
196
  {
197
- "epoch": 1.17,
198
- "learning_rate": 0.000153310927228819,
199
- "loss": 2.5011,
200
- "step": 96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
201
  },
202
  {
203
  "epoch": 1.21,
204
- "learning_rate": 0.00014194306454910757,
205
- "loss": 2.4595,
206
- "step": 99
 
 
 
 
 
 
207
  },
208
  {
209
  "epoch": 1.24,
210
- "learning_rate": 0.00013077658858450138,
211
- "loss": 2.4893,
212
- "step": 102
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
213
  },
214
  {
215
  "epoch": 1.28,
216
- "learning_rate": 0.00011985023337937184,
217
- "loss": 2.4932,
218
- "step": 105
 
 
 
 
 
 
219
  },
220
  {
221
  "epoch": 1.31,
222
- "learning_rate": 0.00010920190005209065,
223
- "loss": 2.4871,
224
- "step": 108
 
 
 
 
 
 
225
  },
226
  {
227
- "epoch": 1.35,
228
- "learning_rate": 9.886852532440312e-05,
229
- "loss": 2.4672,
230
- "step": 111
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
231
  },
232
  {
233
  "epoch": 1.39,
234
- "learning_rate": 8.888595339607961e-05,
235
- "loss": 2.4597,
236
- "step": 114
 
 
 
 
 
 
237
  },
238
  {
239
  "epoch": 1.42,
240
- "learning_rate": 7.928881160928572e-05,
241
- "loss": 2.4531,
242
- "step": 117
243
  },
244
  {
245
- "epoch": 1.46,
246
- "learning_rate": 7.011039033396329e-05,
247
- "loss": 2.4749,
248
- "step": 120
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249
  },
250
  {
251
  "epoch": 1.5,
252
- "learning_rate": 6.138252749087286e-05,
253
- "loss": 2.4551,
254
- "step": 123
 
 
 
 
 
 
255
  },
256
  {
257
  "epoch": 1.53,
258
- "learning_rate": 5.313549811286293e-05,
259
- "loss": 2.4796,
260
- "step": 126
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  {
263
  "epoch": 1.57,
264
- "learning_rate": 4.53979093274526e-05,
265
- "loss": 2.4705,
266
- "step": 129
 
 
 
 
 
 
267
  },
268
  {
269
  "epoch": 1.6,
270
- "learning_rate": 3.819660112501053e-05,
271
- "loss": 2.4915,
272
- "step": 132
273
  },
274
  {
275
- "epoch": 1.64,
276
- "learning_rate": 3.1556553256742713e-05,
277
- "loss": 2.4958,
278
- "step": 135
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
279
  },
280
  {
281
  "epoch": 1.68,
282
- "learning_rate": 2.5500798585440567e-05,
283
- "loss": 2.4814,
284
- "step": 138
 
 
 
 
 
 
285
  },
286
  {
287
  "epoch": 1.71,
288
- "learning_rate": 2.0050343189555743e-05,
289
- "loss": 2.5034,
290
- "step": 141
291
  },
292
  {
293
- "epoch": 1.75,
294
- "learning_rate": 1.5224093497742653e-05,
295
- "loss": 2.4671,
296
- "step": 144
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
297
  },
298
  {
299
  "epoch": 1.79,
300
- "learning_rate": 1.1038790706624391e-05,
301
- "loss": 2.4987,
302
- "step": 147
 
 
 
 
 
 
303
  },
304
  {
305
  "epoch": 1.82,
306
- "learning_rate": 7.508952709270567e-06,
307
- "loss": 2.4618,
308
- "step": 150
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
309
  },
310
  {
311
  "epoch": 1.86,
312
- "learning_rate": 4.646823735825523e-06,
313
- "loss": 2.4815,
314
- "step": 153
 
 
 
 
 
 
315
  },
316
  {
317
  "epoch": 1.89,
318
- "learning_rate": 2.462331880972468e-06,
319
- "loss": 2.4595,
320
- "step": 156
321
  },
322
  {
323
- "epoch": 1.93,
324
- "learning_rate": 9.630546655606364e-07,
325
- "loss": 2.4925,
326
- "step": 159
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
327
  },
328
  {
329
  "epoch": 1.97,
330
- "learning_rate": 1.5419275185541982e-07,
331
- "loss": 2.4333,
332
- "step": 162
333
  },
334
  {
335
  "epoch": 1.99,
336
- "step": 164,
337
- "total_flos": 4.620604962546647e+17,
338
- "train_loss": 2.525294606278582,
339
- "train_runtime": 28715.0205,
340
- "train_samples_per_second": 0.738,
341
- "train_steps_per_second": 0.006
 
 
 
 
 
 
 
 
 
 
 
 
342
  }
343
  ],
344
- "max_steps": 164,
345
  "num_train_epochs": 2,
346
- "total_flos": 4.620604962546647e+17,
347
  "trial_name": null,
348
  "trial_params": null
349
  }
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.9971671388101981,
5
+ "global_step": 330,
6
  "is_hyper_param_search": false,
7
  "is_local_process_zero": true,
8
  "is_world_process_zero": true,
9
  "log_history": [
10
+ {
11
+ "epoch": 0.01,
12
+ "learning_rate": 0.0005,
13
+ "loss": 2.3487,
14
+ "step": 2
15
+ },
16
+ {
17
+ "epoch": 0.02,
18
+ "learning_rate": 0.001,
19
+ "loss": 2.3946,
20
+ "step": 4
21
+ },
22
  {
23
  "epoch": 0.04,
24
+ "learning_rate": 0.0009999071352056674,
25
+ "loss": 2.4059,
26
+ "step": 6
27
+ },
28
+ {
29
+ "epoch": 0.05,
30
+ "learning_rate": 0.00099962857531815,
31
+ "loss": 2.4061,
32
+ "step": 8
33
+ },
34
+ {
35
+ "epoch": 0.06,
36
+ "learning_rate": 0.000999164423811074,
37
+ "loss": 2.3801,
38
+ "step": 10
39
  },
40
  {
41
  "epoch": 0.07,
42
+ "learning_rate": 0.0009985148530977765,
43
+ "loss": 2.4389,
44
+ "step": 12
45
+ },
46
+ {
47
+ "epoch": 0.08,
48
+ "learning_rate": 0.0009976801044672607,
49
+ "loss": 2.4007,
50
+ "step": 14
51
+ },
52
+ {
53
+ "epoch": 0.1,
54
+ "learning_rate": 0.0009966604879945657,
55
+ "loss": 2.4691,
56
+ "step": 16
57
  },
58
  {
59
  "epoch": 0.11,
60
+ "learning_rate": 0.0009954563824255878,
61
+ "loss": 2.4015,
62
+ "step": 18
63
+ },
64
+ {
65
+ "epoch": 0.12,
66
+ "learning_rate": 0.0009940682350363913,
67
+ "loss": 2.4415,
68
+ "step": 20
69
+ },
70
+ {
71
+ "epoch": 0.13,
72
+ "learning_rate": 0.000992496561467063,
73
+ "loss": 2.477,
74
+ "step": 22
75
  },
76
  {
77
  "epoch": 0.15,
78
+ "learning_rate": 0.000990741945530174,
79
+ "loss": 2.4429,
80
+ "step": 24
81
+ },
82
+ {
83
+ "epoch": 0.16,
84
+ "learning_rate": 0.0009888050389939172,
85
+ "loss": 2.4429,
86
+ "step": 26
87
+ },
88
+ {
89
+ "epoch": 0.17,
90
+ "learning_rate": 0.0009866865613400006,
91
+ "loss": 2.4597,
92
+ "step": 28
93
  },
94
  {
95
  "epoch": 0.18,
96
+ "learning_rate": 0.0009843872994963912,
97
+ "loss": 2.4501,
98
+ "step": 30
99
+ },
100
+ {
101
+ "epoch": 0.19,
102
+ "learning_rate": 0.0009819081075450014,
103
+ "loss": 2.4307,
104
+ "step": 32
105
+ },
106
+ {
107
+ "epoch": 0.21,
108
+ "learning_rate": 0.0009792499064044343,
109
+ "loss": 2.4182,
110
+ "step": 34
111
  },
112
  {
113
  "epoch": 0.22,
114
+ "learning_rate": 0.0009764136834878986,
115
+ "loss": 2.4354,
116
+ "step": 36
117
+ },
118
+ {
119
+ "epoch": 0.23,
120
+ "learning_rate": 0.0009734004923364257,
121
+ "loss": 2.4323,
122
+ "step": 38
123
+ },
124
+ {
125
+ "epoch": 0.24,
126
+ "learning_rate": 0.0009702114522275216,
127
+ "loss": 2.4592,
128
+ "step": 40
129
  },
130
  {
131
  "epoch": 0.25,
132
+ "learning_rate": 0.000966847747759402,
133
+ "loss": 2.4242,
134
+ "step": 42
135
+ },
136
+ {
137
+ "epoch": 0.27,
138
+ "learning_rate": 0.0009633106284109611,
139
+ "loss": 2.4355,
140
+ "step": 44
141
+ },
142
+ {
143
+ "epoch": 0.28,
144
+ "learning_rate": 0.0009596014080776422,
145
+ "loss": 2.4379,
146
+ "step": 46
147
  },
148
  {
149
  "epoch": 0.29,
150
+ "learning_rate": 0.0009557214645833791,
151
+ "loss": 2.3786,
152
+ "step": 48
153
+ },
154
+ {
155
+ "epoch": 0.3,
156
+ "learning_rate": 0.0009516722391687902,
157
+ "loss": 2.4303,
158
+ "step": 50
159
+ },
160
+ {
161
+ "epoch": 0.31,
162
+ "learning_rate": 0.0009474552359558167,
163
+ "loss": 2.3946,
164
+ "step": 52
165
  },
166
  {
167
  "epoch": 0.33,
168
+ "learning_rate": 0.000943072021389003,
169
+ "loss": 2.4104,
170
+ "step": 54
171
+ },
172
+ {
173
+ "epoch": 0.34,
174
+ "learning_rate": 0.0009385242236536259,
175
+ "loss": 2.4266,
176
+ "step": 56
177
+ },
178
+ {
179
+ "epoch": 0.35,
180
+ "learning_rate": 0.0009338135320708912,
181
+ "loss": 2.5106,
182
+ "step": 58
183
  },
184
  {
185
  "epoch": 0.36,
186
+ "learning_rate": 0.0009289416964704185,
187
+ "loss": 2.4225,
188
+ "step": 60
189
+ },
190
+ {
191
+ "epoch": 0.37,
192
+ "learning_rate": 0.0009239105265402525,
193
+ "loss": 2.4745,
194
+ "step": 62
195
+ },
196
+ {
197
+ "epoch": 0.39,
198
+ "learning_rate": 0.0009187218911546363,
199
+ "loss": 2.4572,
200
+ "step": 64
201
  },
202
  {
203
  "epoch": 0.4,
204
+ "learning_rate": 0.0009133777176798013,
205
+ "loss": 2.4366,
206
+ "step": 66
207
+ },
208
+ {
209
+ "epoch": 0.41,
210
+ "learning_rate": 0.0009078799912580304,
211
+ "loss": 2.4021,
212
+ "step": 68
213
+ },
214
+ {
215
+ "epoch": 0.42,
216
+ "learning_rate": 0.0009022307540702576,
217
+ "loss": 2.4054,
218
+ "step": 70
219
  },
220
  {
221
  "epoch": 0.44,
222
+ "learning_rate": 0.0008964321045774807,
223
+ "loss": 2.4628,
224
+ "step": 72
225
+ },
226
+ {
227
+ "epoch": 0.45,
228
+ "learning_rate": 0.0008904861967412702,
229
+ "loss": 2.5038,
230
+ "step": 74
231
+ },
232
+ {
233
+ "epoch": 0.46,
234
+ "learning_rate": 0.0008843952392236594,
235
+ "loss": 2.3801,
236
+ "step": 76
237
  },
238
  {
239
  "epoch": 0.47,
240
+ "learning_rate": 0.0008781614945667169,
241
+ "loss": 2.4056,
242
+ "step": 78
243
+ },
244
+ {
245
+ "epoch": 0.48,
246
+ "learning_rate": 0.0008717872783521047,
247
+ "loss": 2.3334,
248
+ "step": 80
249
+ },
250
+ {
251
+ "epoch": 0.5,
252
+ "learning_rate": 0.0008652749583409339,
253
+ "loss": 2.3913,
254
+ "step": 82
255
  },
256
  {
257
  "epoch": 0.51,
258
+ "learning_rate": 0.0008586269535942384,
259
+ "loss": 2.3784,
260
+ "step": 84
261
+ },
262
+ {
263
+ "epoch": 0.52,
264
+ "learning_rate": 0.0008518457335743926,
265
+ "loss": 2.4436,
266
+ "step": 86
267
+ },
268
+ {
269
+ "epoch": 0.53,
270
+ "learning_rate": 0.0008449338172278058,
271
+ "loss": 2.3735,
272
+ "step": 88
273
  },
274
  {
275
  "epoch": 0.54,
276
+ "learning_rate": 0.0008378937720492384,
277
+ "loss": 2.374,
278
+ "step": 90
279
+ },
280
+ {
281
+ "epoch": 0.56,
282
+ "learning_rate": 0.0008307282131280805,
283
+ "loss": 2.4064,
284
+ "step": 92
285
+ },
286
+ {
287
+ "epoch": 0.57,
288
+ "learning_rate": 0.000823439802176954,
289
+ "loss": 2.4124,
290
+ "step": 94
291
  },
292
  {
293
  "epoch": 0.58,
294
+ "learning_rate": 0.0008160312465429952,
295
+ "loss": 2.4181,
296
+ "step": 96
297
+ },
298
+ {
299
+ "epoch": 0.59,
300
+ "learning_rate": 0.0008085052982021848,
301
+ "loss": 2.4253,
302
+ "step": 98
303
+ },
304
+ {
305
+ "epoch": 0.6,
306
+ "learning_rate": 0.0008008647527371022,
307
+ "loss": 2.4678,
308
+ "step": 100
309
  },
310
  {
311
  "epoch": 0.62,
312
+ "learning_rate": 0.0007931124482984802,
313
+ "loss": 2.4738,
314
+ "step": 102
315
+ },
316
+ {
317
+ "epoch": 0.63,
318
+ "learning_rate": 0.0007852512645509479,
319
+ "loss": 2.3738,
320
+ "step": 104
321
+ },
322
+ {
323
+ "epoch": 0.64,
324
+ "learning_rate": 0.0007772841216033533,
325
+ "loss": 2.4081,
326
+ "step": 106
327
  },
328
  {
329
  "epoch": 0.65,
330
+ "learning_rate": 0.0007692139789240611,
331
+ "loss": 2.3738,
332
+ "step": 108
333
+ },
334
+ {
335
+ "epoch": 0.66,
336
+ "learning_rate": 0.0007610438342416319,
337
+ "loss": 2.3701,
338
+ "step": 110
339
+ },
340
+ {
341
+ "epoch": 0.68,
342
+ "learning_rate": 0.0007527767224312882,
343
+ "loss": 2.4355,
344
+ "step": 112
345
  },
346
  {
347
  "epoch": 0.69,
348
+ "learning_rate": 0.000744415714387582,
349
+ "loss": 2.4036,
350
+ "step": 114
351
+ },
352
+ {
353
+ "epoch": 0.7,
354
+ "learning_rate": 0.0007359639158836828,
355
+ "loss": 2.3746,
356
+ "step": 116
357
+ },
358
+ {
359
+ "epoch": 0.71,
360
+ "learning_rate": 0.0007274244664177097,
361
+ "loss": 2.4855,
362
+ "step": 118
363
  },
364
  {
365
  "epoch": 0.73,
366
+ "learning_rate": 0.0007188005380465365,
367
+ "loss": 2.379,
368
+ "step": 120
369
+ },
370
+ {
371
+ "epoch": 0.74,
372
+ "learning_rate": 0.000710095334207501,
373
+ "loss": 2.4178,
374
+ "step": 122
375
+ },
376
+ {
377
+ "epoch": 0.75,
378
+ "learning_rate": 0.0007013120885284599,
379
+ "loss": 2.4561,
380
+ "step": 124
381
  },
382
  {
383
  "epoch": 0.76,
384
+ "learning_rate": 0.0006924540636266272,
385
+ "loss": 2.4024,
386
+ "step": 126
387
+ },
388
+ {
389
+ "epoch": 0.77,
390
+ "learning_rate": 0.000683524549896646,
391
+ "loss": 2.4172,
392
+ "step": 128
393
+ },
394
+ {
395
+ "epoch": 0.79,
396
+ "learning_rate": 0.0006745268642883404,
397
+ "loss": 2.3858,
398
+ "step": 130
399
  },
400
  {
401
  "epoch": 0.8,
402
+ "learning_rate": 0.0006654643490746042,
403
+ "loss": 2.3547,
404
+ "step": 132
405
+ },
406
+ {
407
+ "epoch": 0.81,
408
+ "learning_rate": 0.0006563403706098833,
409
+ "loss": 2.4372,
410
+ "step": 134
411
+ },
412
+ {
413
+ "epoch": 0.82,
414
+ "learning_rate": 0.0006471583180797121,
415
+ "loss": 2.3785,
416
+ "step": 136
417
  },
418
  {
419
  "epoch": 0.83,
420
+ "learning_rate": 0.0006379216022417695,
421
+ "loss": 2.3815,
422
+ "step": 138
423
+ },
424
+ {
425
+ "epoch": 0.85,
426
+ "learning_rate": 0.0006286336541589224,
427
+ "loss": 2.4209,
428
+ "step": 140
429
+ },
430
+ {
431
+ "epoch": 0.86,
432
+ "learning_rate": 0.0006192979239247243,
433
+ "loss": 2.3962,
434
+ "step": 142
435
  },
436
  {
437
  "epoch": 0.87,
438
+ "learning_rate": 0.0006099178793818478,
439
+ "loss": 2.3626,
440
+ "step": 144
441
+ },
442
+ {
443
+ "epoch": 0.88,
444
+ "learning_rate": 0.0006004970048339225,
445
+ "loss": 2.3991,
446
+ "step": 146
447
+ },
448
+ {
449
+ "epoch": 0.89,
450
+ "learning_rate": 0.0005910387997512573,
451
+ "loss": 2.4396,
452
+ "step": 148
453
  },
454
  {
455
  "epoch": 0.91,
456
+ "learning_rate": 0.0005815467774709313,
457
+ "loss": 2.3816,
458
+ "step": 150
459
+ },
460
+ {
461
+ "epoch": 0.92,
462
+ "learning_rate": 0.0005720244638917323,
463
+ "loss": 2.3866,
464
+ "step": 152
465
+ },
466
+ {
467
+ "epoch": 0.93,
468
+ "learning_rate": 0.0005624753961644281,
469
+ "loss": 2.4035,
470
+ "step": 154
471
  },
472
  {
473
  "epoch": 0.94,
474
+ "learning_rate": 0.0005529031213778615,
475
+ "loss": 2.4063,
476
+ "step": 156
477
+ },
478
+ {
479
+ "epoch": 0.95,
480
+ "learning_rate": 0.0005433111952413496,
481
+ "loss": 2.3944,
482
+ "step": 158
483
+ },
484
+ {
485
+ "epoch": 0.97,
486
+ "learning_rate": 0.0005337031807638841,
487
+ "loss": 2.4192,
488
+ "step": 160
489
  },
490
  {
491
  "epoch": 0.98,
492
+ "learning_rate": 0.0005240826469306187,
493
+ "loss": 2.3603,
494
+ "step": 162
495
+ },
496
+ {
497
+ "epoch": 0.99,
498
+ "learning_rate": 0.0005144531673771364,
499
+ "loss": 2.4041,
500
+ "step": 164
501
+ },
502
+ {
503
+ "epoch": 1.01,
504
+ "learning_rate": 0.0005048183190619903,
505
+ "loss": 2.8813,
506
+ "step": 166
507
  },
508
  {
509
  "epoch": 1.02,
510
+ "learning_rate": 0.0004951816809380097,
511
+ "loss": 2.2786,
512
+ "step": 168
513
+ },
514
+ {
515
+ "epoch": 1.03,
516
+ "learning_rate": 0.0004855468326228638,
517
+ "loss": 2.2886,
518
+ "step": 170
519
  },
520
  {
521
+ "epoch": 1.04,
522
+ "learning_rate": 0.00047591735306938137,
523
+ "loss": 2.1822,
524
+ "step": 172
525
+ },
526
+ {
527
+ "epoch": 1.05,
528
+ "learning_rate": 0.00046629681923611606,
529
+ "loss": 2.2589,
530
+ "step": 174
531
+ },
532
+ {
533
+ "epoch": 1.07,
534
+ "learning_rate": 0.0004566888047586507,
535
+ "loss": 2.2625,
536
+ "step": 176
537
+ },
538
+ {
539
+ "epoch": 1.08,
540
+ "learning_rate": 0.00044709687862213866,
541
+ "loss": 2.2715,
542
+ "step": 178
543
+ },
544
+ {
545
+ "epoch": 1.09,
546
+ "learning_rate": 0.000437524603835572,
547
+ "loss": 2.1988,
548
+ "step": 180
549
  },
550
  {
551
  "epoch": 1.1,
552
+ "learning_rate": 0.000427975536108268,
553
+ "loss": 2.3257,
554
+ "step": 182
555
+ },
556
+ {
557
+ "epoch": 1.11,
558
+ "learning_rate": 0.00041845322252906863,
559
+ "loss": 2.3026,
560
+ "step": 184
561
  },
562
  {
563
  "epoch": 1.13,
564
+ "learning_rate": 0.00040896120024874283,
565
+ "loss": 2.2306,
566
+ "step": 186
567
  },
568
  {
569
+ "epoch": 1.14,
570
+ "learning_rate": 0.0003995029951660776,
571
+ "loss": 2.2269,
572
+ "step": 188
573
+ },
574
+ {
575
+ "epoch": 1.15,
576
+ "learning_rate": 0.00039008212061815206,
577
+ "loss": 2.3079,
578
+ "step": 190
579
+ },
580
+ {
581
+ "epoch": 1.16,
582
+ "learning_rate": 0.00038070207607527587,
583
+ "loss": 2.218,
584
+ "step": 192
585
+ },
586
+ {
587
+ "epoch": 1.18,
588
+ "learning_rate": 0.00037136634584107787,
589
+ "loss": 2.2667,
590
+ "step": 194
591
+ },
592
+ {
593
+ "epoch": 1.19,
594
+ "learning_rate": 0.0003620783977582305,
595
+ "loss": 2.2754,
596
+ "step": 196
597
+ },
598
+ {
599
+ "epoch": 1.2,
600
+ "learning_rate": 0.0003528416819202881,
601
+ "loss": 2.2835,
602
+ "step": 198
603
  },
604
  {
605
  "epoch": 1.21,
606
+ "learning_rate": 0.00034365962939011697,
607
+ "loss": 2.2843,
608
+ "step": 200
609
+ },
610
+ {
611
+ "epoch": 1.22,
612
+ "learning_rate": 0.00033453565092539584,
613
+ "loss": 2.2387,
614
+ "step": 202
615
  },
616
  {
617
  "epoch": 1.24,
618
+ "learning_rate": 0.0003254731357116597,
619
+ "loss": 2.254,
620
+ "step": 204
621
+ },
622
+ {
623
+ "epoch": 1.25,
624
+ "learning_rate": 0.000316475450103354,
625
+ "loss": 2.2686,
626
+ "step": 206
627
+ },
628
+ {
629
+ "epoch": 1.26,
630
+ "learning_rate": 0.00030754593637337277,
631
+ "loss": 2.2422,
632
+ "step": 208
633
+ },
634
+ {
635
+ "epoch": 1.27,
636
+ "learning_rate": 0.0002986879114715403,
637
+ "loss": 2.3003,
638
+ "step": 210
639
  },
640
  {
641
  "epoch": 1.28,
642
+ "learning_rate": 0.0002899046657924992,
643
+ "loss": 2.2619,
644
+ "step": 212
645
+ },
646
+ {
647
+ "epoch": 1.3,
648
+ "learning_rate": 0.00028119946195346375,
649
+ "loss": 2.3022,
650
+ "step": 214
651
  },
652
  {
653
  "epoch": 1.31,
654
+ "learning_rate": 0.00027257553358229033,
655
+ "loss": 2.2523,
656
+ "step": 216
657
+ },
658
+ {
659
+ "epoch": 1.32,
660
+ "learning_rate": 0.0002640360841163174,
661
+ "loss": 2.3098,
662
+ "step": 218
663
  },
664
  {
665
+ "epoch": 1.33,
666
+ "learning_rate": 0.0002555842856124182,
667
+ "loss": 2.235,
668
+ "step": 220
669
+ },
670
+ {
671
+ "epoch": 1.34,
672
+ "learning_rate": 0.00024722327756871186,
673
+ "loss": 2.2448,
674
+ "step": 222
675
+ },
676
+ {
677
+ "epoch": 1.36,
678
+ "learning_rate": 0.0002389561657583681,
679
+ "loss": 2.2411,
680
+ "step": 224
681
+ },
682
+ {
683
+ "epoch": 1.37,
684
+ "learning_rate": 0.00023078602107593898,
685
+ "loss": 2.2485,
686
+ "step": 226
687
+ },
688
+ {
689
+ "epoch": 1.38,
690
+ "learning_rate": 0.0002227158783966467,
691
+ "loss": 2.2261,
692
+ "step": 228
693
  },
694
  {
695
  "epoch": 1.39,
696
+ "learning_rate": 0.00021474873544905204,
697
+ "loss": 2.2427,
698
+ "step": 230
699
+ },
700
+ {
701
+ "epoch": 1.4,
702
+ "learning_rate": 0.00020688755170151997,
703
+ "loss": 2.2961,
704
+ "step": 232
705
  },
706
  {
707
  "epoch": 1.42,
708
+ "learning_rate": 0.00019913524726289784,
709
+ "loss": 2.2272,
710
+ "step": 234
711
  },
712
  {
713
+ "epoch": 1.43,
714
+ "learning_rate": 0.00019149470179781532,
715
+ "loss": 2.2368,
716
+ "step": 236
717
+ },
718
+ {
719
+ "epoch": 1.44,
720
+ "learning_rate": 0.00018396875345700497,
721
+ "loss": 2.2846,
722
+ "step": 238
723
+ },
724
+ {
725
+ "epoch": 1.45,
726
+ "learning_rate": 0.000176560197823046,
727
+ "loss": 2.1709,
728
+ "step": 240
729
+ },
730
+ {
731
+ "epoch": 1.47,
732
+ "learning_rate": 0.0001692717868719195,
733
+ "loss": 2.2659,
734
+ "step": 242
735
+ },
736
+ {
737
+ "epoch": 1.48,
738
+ "learning_rate": 0.0001621062279507617,
739
+ "loss": 2.2655,
740
+ "step": 244
741
+ },
742
+ {
743
+ "epoch": 1.49,
744
+ "learning_rate": 0.0001550661827721941,
745
+ "loss": 2.2284,
746
+ "step": 246
747
  },
748
  {
749
  "epoch": 1.5,
750
+ "learning_rate": 0.00014815426642560752,
751
+ "loss": 2.2444,
752
+ "step": 248
753
+ },
754
+ {
755
+ "epoch": 1.51,
756
+ "learning_rate": 0.0001413730464057616,
757
+ "loss": 2.3102,
758
+ "step": 250
759
  },
760
  {
761
  "epoch": 1.53,
762
+ "learning_rate": 0.00013472504165906613,
763
+ "loss": 2.2287,
764
+ "step": 252
765
+ },
766
+ {
767
+ "epoch": 1.54,
768
+ "learning_rate": 0.00012821272164789544,
769
+ "loss": 2.2713,
770
+ "step": 254
771
+ },
772
+ {
773
+ "epoch": 1.55,
774
+ "learning_rate": 0.00012183850543328313,
775
+ "loss": 2.2127,
776
+ "step": 256
777
+ },
778
+ {
779
+ "epoch": 1.56,
780
+ "learning_rate": 0.00011560476077634069,
781
+ "loss": 2.1682,
782
+ "step": 258
783
  },
784
  {
785
  "epoch": 1.57,
786
+ "learning_rate": 0.00010951380325872979,
787
+ "loss": 2.2393,
788
+ "step": 260
789
+ },
790
+ {
791
+ "epoch": 1.59,
792
+ "learning_rate": 0.00010356789542251938,
793
+ "loss": 2.2259,
794
+ "step": 262
795
  },
796
  {
797
  "epoch": 1.6,
798
+ "learning_rate": 9.776924592974257e-05,
799
+ "loss": 2.2157,
800
+ "step": 264
801
  },
802
  {
803
+ "epoch": 1.61,
804
+ "learning_rate": 9.212000874196952e-05,
805
+ "loss": 2.2393,
806
+ "step": 266
807
+ },
808
+ {
809
+ "epoch": 1.62,
810
+ "learning_rate": 8.662228232019875e-05,
811
+ "loss": 2.2613,
812
+ "step": 268
813
+ },
814
+ {
815
+ "epoch": 1.63,
816
+ "learning_rate": 8.127810884536401e-05,
817
+ "loss": 2.1981,
818
+ "step": 270
819
+ },
820
+ {
821
+ "epoch": 1.65,
822
+ "learning_rate": 7.60894734597476e-05,
823
+ "loss": 2.2457,
824
+ "step": 272
825
+ },
826
+ {
827
+ "epoch": 1.66,
828
+ "learning_rate": 7.105830352958143e-05,
829
+ "loss": 2.2571,
830
+ "step": 274
831
+ },
832
+ {
833
+ "epoch": 1.67,
834
+ "learning_rate": 6.618646792910893e-05,
835
+ "loss": 2.1771,
836
+ "step": 276
837
  },
838
  {
839
  "epoch": 1.68,
840
+ "learning_rate": 6.147577634637414e-05,
841
+ "loss": 2.2243,
842
+ "step": 278
843
+ },
844
+ {
845
+ "epoch": 1.69,
846
+ "learning_rate": 5.692797861099719e-05,
847
+ "loss": 2.2427,
848
+ "step": 280
849
  },
850
  {
851
  "epoch": 1.71,
852
+ "learning_rate": 5.25447640441834e-05,
853
+ "loss": 2.2266,
854
+ "step": 282
855
  },
856
  {
857
+ "epoch": 1.72,
858
+ "learning_rate": 4.832776083120982e-05,
859
+ "loss": 2.3057,
860
+ "step": 284
861
+ },
862
+ {
863
+ "epoch": 1.73,
864
+ "learning_rate": 4.4278535416620916e-05,
865
+ "loss": 2.2225,
866
+ "step": 286
867
+ },
868
+ {
869
+ "epoch": 1.74,
870
+ "learning_rate": 4.039859192235778e-05,
871
+ "loss": 2.2665,
872
+ "step": 288
873
+ },
874
+ {
875
+ "epoch": 1.76,
876
+ "learning_rate": 3.668937158903901e-05,
877
+ "loss": 2.2807,
878
+ "step": 290
879
+ },
880
+ {
881
+ "epoch": 1.77,
882
+ "learning_rate": 3.315225224059809e-05,
883
+ "loss": 2.2165,
884
+ "step": 292
885
+ },
886
+ {
887
+ "epoch": 1.78,
888
+ "learning_rate": 2.9788547772478415e-05,
889
+ "loss": 2.2651,
890
+ "step": 294
891
  },
892
  {
893
  "epoch": 1.79,
894
+ "learning_rate": 2.6599507663574384e-05,
895
+ "loss": 2.2437,
896
+ "step": 296
897
+ },
898
+ {
899
+ "epoch": 1.8,
900
+ "learning_rate": 2.3586316512101414e-05,
901
+ "loss": 2.3066,
902
+ "step": 298
903
  },
904
  {
905
  "epoch": 1.82,
906
+ "learning_rate": 2.0750093595565732e-05,
907
+ "loss": 2.1727,
908
+ "step": 300
909
+ },
910
+ {
911
+ "epoch": 1.83,
912
+ "learning_rate": 1.8091892454998595e-05,
913
+ "loss": 2.2409,
914
+ "step": 302
915
+ },
916
+ {
917
+ "epoch": 1.84,
918
+ "learning_rate": 1.561270050360897e-05,
919
+ "loss": 2.2908,
920
+ "step": 304
921
+ },
922
+ {
923
+ "epoch": 1.85,
924
+ "learning_rate": 1.33134386599994e-05,
925
+ "loss": 2.2925,
926
+ "step": 306
927
  },
928
  {
929
  "epoch": 1.86,
930
+ "learning_rate": 1.1194961006082971e-05,
931
+ "loss": 2.2449,
932
+ "step": 308
933
+ },
934
+ {
935
+ "epoch": 1.88,
936
+ "learning_rate": 9.258054469825972e-06,
937
+ "loss": 2.235,
938
+ "step": 310
939
  },
940
  {
941
  "epoch": 1.89,
942
+ "learning_rate": 7.503438532937168e-06,
943
+ "loss": 2.2216,
944
+ "step": 312
945
  },
946
  {
947
+ "epoch": 1.9,
948
+ "learning_rate": 5.931764963608866e-06,
949
+ "loss": 2.2884,
950
+ "step": 314
951
+ },
952
+ {
953
+ "epoch": 1.91,
954
+ "learning_rate": 4.5436175744121845e-06,
955
+ "loss": 2.2124,
956
+ "step": 316
957
+ },
958
+ {
959
+ "epoch": 1.92,
960
+ "learning_rate": 3.3395120054343087e-06,
961
+ "loss": 2.2418,
962
+ "step": 318
963
+ },
964
+ {
965
+ "epoch": 1.94,
966
+ "learning_rate": 2.319895532739369e-06,
967
+ "loss": 2.2855,
968
+ "step": 320
969
+ },
970
+ {
971
+ "epoch": 1.95,
972
+ "learning_rate": 1.4851469022234e-06,
973
+ "loss": 2.2974,
974
+ "step": 322
975
+ },
976
+ {
977
+ "epoch": 1.96,
978
+ "learning_rate": 8.35576188926046e-07,
979
+ "loss": 2.2552,
980
+ "step": 324
981
  },
982
  {
983
  "epoch": 1.97,
984
+ "learning_rate": 3.71424681850141e-07,
985
+ "loss": 2.2209,
986
+ "step": 326
987
  },
988
  {
989
  "epoch": 1.99,
990
+ "learning_rate": 9.286479433257e-08,
991
+ "loss": 2.1935,
992
+ "step": 328
993
+ },
994
+ {
995
+ "epoch": 2.0,
996
+ "learning_rate": 0.0,
997
+ "loss": 2.2702,
998
+ "step": 330
999
+ },
1000
+ {
1001
+ "epoch": 2.0,
1002
+ "step": 330,
1003
+ "total_flos": 4.634629374287544e+17,
1004
+ "train_loss": 2.336302039117524,
1005
+ "train_runtime": 79791.9217,
1006
+ "train_samples_per_second": 0.265,
1007
+ "train_steps_per_second": 0.004
1008
  }
1009
  ],
1010
+ "max_steps": 330,
1011
  "num_train_epochs": 2,
1012
+ "total_flos": 4.634629374287544e+17,
1013
  "trial_name": null,
1014
  "trial_params": null
1015
  }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:673e02f4e38479f806c1a20434ea2efa4f9cf7a6aa07067907fde9dc0160405c
3
  size 4527
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4526ccf2486e6fb3048af4d26eb6228cf640199b02d5c9ab46e06e3bf549ec3a
3
  size 4527
zero_to_fp32.py ADDED
@@ -0,0 +1,484 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # This script extracts fp32 consolidated weights from a zero 2 and 3 DeepSpeed checkpoints. It gets
4
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
5
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
6
+ # application.
7
+ #
8
+ # example: python zero_to_fp32.py . pytorch_model.bin
9
+
10
+ import argparse
11
+ import torch
12
+ import glob
13
+ import math
14
+ import os
15
+ import re
16
+ from collections import OrderedDict
17
+
18
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
19
+ # DeepSpeed data structures it has to be available in the current python environment.
20
+ import deepspeed
21
+ from deepspeed.utils import logger
22
+ from deepspeed.checkpoint.constants import (DS_VERSION,
23
+ OPTIMIZER_STATE_DICT,
24
+ PARAM_SHAPES,
25
+ SINGLE_PARTITION_OF_FP32_GROUPS,
26
+ FP32_FLAT_GROUPS,
27
+ ZERO_STAGE,
28
+ PARTITION_COUNT,
29
+ PARAM_SHAPES,
30
+ BUFFER_NAMES)
31
+
32
+ debug = 0
33
+
34
+ # load to cpu
35
+ device = torch.device('cpu')
36
+
37
+
38
+ def atoi(text):
39
+ return int(text) if text.isdigit() else text
40
+
41
+
42
+ def natural_keys(text):
43
+ '''
44
+ alist.sort(key=natural_keys) sorts in human order
45
+ http://nedbatchelder.com/blog/200712/human_sorting.html
46
+ (See Toothy's implementation in the comments)
47
+ '''
48
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
49
+
50
+
51
+ def get_model_state_file(checkpoint_dir, zero_stage):
52
+ if not os.path.isdir(checkpoint_dir):
53
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
54
+
55
+ # there should be only one file
56
+ if zero_stage == 2:
57
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
58
+ elif zero_stage == 3:
59
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
60
+
61
+ if not os.path.exists(file):
62
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
63
+
64
+ return file
65
+
66
+
67
+ def get_optim_files(checkpoint_dir):
68
+ # XXX: need to test that this simple glob rule works for multi-node setup too
69
+ optim_files = sorted(glob.glob(os.path.join(checkpoint_dir,
70
+ "*_optim_states.pt")),
71
+ key=natural_keys)
72
+
73
+ if len(optim_files) == 0:
74
+ raise FileNotFoundError(
75
+ f"can't find '*_optim_states.pt' files in directory '{checkpoint_dir}'")
76
+
77
+ return optim_files
78
+
79
+
80
+ def parse_model_state(file):
81
+ state_dict = torch.load(file, map_location=device)
82
+
83
+ if BUFFER_NAMES not in state_dict:
84
+ raise ValueError(f"{file} is not a model state checkpoint")
85
+ buffer_names = state_dict[BUFFER_NAMES]
86
+ if debug:
87
+ print("Found buffers:", buffer_names)
88
+
89
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
90
+ buffers = {
91
+ k: v.float()
92
+ for k,
93
+ v in state_dict["module"].items() if k in buffer_names
94
+ }
95
+ param_shapes = state_dict[PARAM_SHAPES]
96
+
97
+ ds_version = state_dict.get(DS_VERSION, None)
98
+
99
+ return buffers, param_shapes, ds_version
100
+
101
+
102
+ def parse_optim_states(files, ds_checkpoint_dir):
103
+
104
+ total_files = len(files)
105
+ state_dicts = []
106
+ for f in files:
107
+ state_dicts.append(torch.load(f, map_location=device))
108
+
109
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
110
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
111
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
112
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
113
+
114
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
115
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
116
+ # use the max of the partition_count to get the dp world_size.
117
+
118
+ if type(world_size) is list:
119
+ world_size = max(world_size)
120
+
121
+ if world_size != total_files:
122
+ raise ValueError(
123
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
124
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
125
+ )
126
+
127
+ # the groups are named differently in each stage
128
+ if zero_stage == 2:
129
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
130
+ elif zero_stage == 3:
131
+ fp32_groups_key = FP32_FLAT_GROUPS
132
+ else:
133
+ raise ValueError(f"unknown zero stage {zero_stage}")
134
+
135
+ if zero_stage == 2:
136
+ fp32_flat_groups = [
137
+ state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key]
138
+ for i in range(len(state_dicts))
139
+ ]
140
+ elif zero_stage == 3:
141
+ # if there is more than one param group, there will be multiple flattened tensors - one
142
+ # flattened tensor per group - for simplicity merge them into a single tensor
143
+ #
144
+ # XXX: could make the script more memory efficient for when there are multiple groups - it
145
+ # will require matching the sub-lists of param_shapes for each param group flattened tensor
146
+
147
+ fp32_flat_groups = [
148
+ torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key],
149
+ 0) for i in range(len(state_dicts))
150
+ ]
151
+
152
+ return zero_stage, world_size, fp32_flat_groups
153
+
154
+
155
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
156
+ """
157
+ Returns fp32 state_dict reconstructed from ds checkpoint
158
+
159
+ Args:
160
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
161
+
162
+ """
163
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
164
+
165
+ optim_files = get_optim_files(ds_checkpoint_dir)
166
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
167
+ print(
168
+ f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
169
+
170
+ model_file = get_model_state_file(ds_checkpoint_dir, zero_stage)
171
+ buffers, param_shapes, ds_version = parse_model_state(model_file)
172
+ print(f'Parsing checkpoint created by deepspeed=={ds_version}')
173
+
174
+ if zero_stage == 2:
175
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size,
176
+ param_shapes,
177
+ fp32_flat_groups,
178
+ buffers)
179
+ elif zero_stage == 3:
180
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size,
181
+ param_shapes,
182
+ fp32_flat_groups,
183
+ buffers)
184
+
185
+
186
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
187
+ param_shapes,
188
+ fp32_flat_groups,
189
+ buffers):
190
+
191
+ # Reconstruction protocol:
192
+ #
193
+ # XXX: document this
194
+
195
+ if debug:
196
+ for i in range(world_size):
197
+ for j in range(len(fp32_flat_groups[0])):
198
+ print(
199
+ f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
200
+
201
+ # XXX: memory usage doubles here (zero2)
202
+ num_param_groups = len(fp32_flat_groups[0])
203
+ merged_single_partition_of_fp32_groups = []
204
+ for i in range(num_param_groups):
205
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
206
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
207
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
208
+ avail_numel = sum([
209
+ full_single_fp32_vector.numel()
210
+ for full_single_fp32_vector in merged_single_partition_of_fp32_groups
211
+ ])
212
+
213
+ if debug:
214
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
215
+ wanted_numel = sum(
216
+ [sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
217
+ # not asserting if there is a mismatch due to possible padding
218
+ print(f"Have {avail_numel} numels to process.")
219
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
220
+
221
+ state_dict = OrderedDict()
222
+
223
+ # buffers
224
+ state_dict.update(buffers)
225
+ if debug:
226
+ print(f"added {len(buffers)} buffers")
227
+
228
+ # params
229
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
230
+ # out-of-core computing solution
231
+ total_numel = 0
232
+ total_params = 0
233
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
234
+ offset = 0
235
+ avail_numel = full_single_fp32_vector.numel()
236
+ for name, shape in shapes.items():
237
+
238
+ unpartitioned_numel = shape.numel()
239
+ total_numel += unpartitioned_numel
240
+ total_params += 1
241
+
242
+ if debug:
243
+ print(
244
+ f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} "
245
+ )
246
+ state_dict[name] = full_single_fp32_vector.narrow(
247
+ 0,
248
+ offset,
249
+ unpartitioned_numel).view(shape)
250
+ offset += unpartitioned_numel
251
+
252
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
253
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
254
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
255
+ # live optimizer object, so we are checking that the numbers are within the right range
256
+ align_to = 2 * world_size
257
+
258
+ def zero2_align(x):
259
+ return align_to * math.ceil(x / align_to)
260
+
261
+ if debug:
262
+ print(f"original offset={offset}, avail_numel={avail_numel}")
263
+
264
+ offset = zero2_align(offset)
265
+ avail_numel = zero2_align(avail_numel)
266
+
267
+ if debug:
268
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
269
+
270
+ # Sanity check
271
+ if offset != avail_numel:
272
+ raise ValueError(
273
+ f"consumed {offset} numels out of {avail_numel} - something is wrong")
274
+
275
+ print(
276
+ f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements"
277
+ )
278
+
279
+ return state_dict
280
+
281
+
282
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
283
+ remainder = unpartitioned_numel % world_size
284
+ padding_numel = (world_size - remainder) if remainder else 0
285
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
286
+ return partitioned_numel, padding_numel
287
+
288
+
289
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size,
290
+ param_shapes,
291
+ fp32_flat_groups,
292
+ buffers):
293
+
294
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
295
+ # param, re-consolidating each param, while dealing with padding if any
296
+
297
+ avail_numel = fp32_flat_groups[0].numel() * world_size
298
+ # merge list of dicts, preserving order
299
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
300
+
301
+ if debug:
302
+ for i in range(world_size):
303
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
304
+
305
+ wanted_params = len(param_shapes)
306
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
307
+ # not asserting if there is a mismatch due to possible padding
308
+ print(f"Have {avail_numel} numels to process.")
309
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
310
+
311
+ state_dict = OrderedDict()
312
+
313
+ # buffers
314
+ state_dict.update(buffers)
315
+ if debug:
316
+ print(f"added {len(buffers)} buffers")
317
+
318
+ # params
319
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
320
+ # out-of-core computing solution
321
+ offset = 0
322
+ total_numel = 0
323
+ total_params = 0
324
+ for name, shape in param_shapes.items():
325
+
326
+ unpartitioned_numel = shape.numel()
327
+ total_numel += unpartitioned_numel
328
+ total_params += 1
329
+
330
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
331
+
332
+ if debug:
333
+ print(
334
+ f"{total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
335
+ )
336
+
337
+ # XXX: memory usage doubles here
338
+ state_dict[name] = torch.cat(
339
+ tuple(fp32_flat_groups[i].narrow(0,
340
+ offset,
341
+ partitioned_numel)
342
+ for i in range(world_size)),
343
+ 0).narrow(0,
344
+ 0,
345
+ unpartitioned_numel).view(shape)
346
+ offset += partitioned_numel
347
+
348
+ offset *= world_size
349
+
350
+ # Sanity check
351
+ if offset != avail_numel:
352
+ raise ValueError(
353
+ f"consumed {offset} numels out of {avail_numel} - something is wrong")
354
+
355
+ print(
356
+ f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements"
357
+ )
358
+
359
+ return state_dict
360
+
361
+
362
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
363
+ """
364
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
365
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
366
+ via a model hub.
367
+
368
+ Args:
369
+ - ``checkpoint_dir``: path to the desired checkpoint folder
370
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
371
+
372
+ Returns:
373
+ - pytorch ``state_dict``
374
+
375
+ Note: this approach may not work if your application doesn't have sufficient free CPU memory and
376
+ you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
377
+ the checkpoint.
378
+
379
+ A typical usage might be ::
380
+
381
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
382
+ # do the training and checkpoint saving
383
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
384
+ model = model.cpu() # move to cpu
385
+ model.load_state_dict(state_dict)
386
+ # submit to model hub or save the model to share with others
387
+
388
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
389
+ application. i.e. you will need to re-initialize the deepspeed engine, since
390
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
391
+
392
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
393
+
394
+ """
395
+ if tag is None:
396
+ latest_path = os.path.join(checkpoint_dir, 'latest')
397
+ if os.path.isfile(latest_path):
398
+ with open(latest_path, 'r') as fd:
399
+ tag = fd.read().strip()
400
+ else:
401
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
402
+
403
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
404
+
405
+ if not os.path.isdir(ds_checkpoint_dir):
406
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
407
+
408
+ return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
409
+
410
+
411
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
412
+ """
413
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
414
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
415
+
416
+ Args:
417
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
418
+ - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
419
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
420
+ """
421
+
422
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
423
+ print(f"Saving fp32 state dict to {output_file}")
424
+ torch.save(state_dict, output_file)
425
+
426
+
427
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
428
+ """
429
+ 1. Put the provided model to cpu
430
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
431
+ 3. Load it into the provided model
432
+
433
+ Args:
434
+ - ``model``: the model object to update
435
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
436
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
437
+
438
+ Returns:
439
+ - ``model`: modified model
440
+
441
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
442
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
443
+ conveniently placed for you in the checkpoint folder.
444
+
445
+ A typical usage might be ::
446
+
447
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
448
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
449
+ # submit to model hub or save the model to share with others
450
+
451
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
452
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
453
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
454
+
455
+ """
456
+ logger.info(f"Extracting fp32 weights")
457
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
458
+
459
+ logger.info(f"Overwriting model with fp32 weights")
460
+ model = model.cpu()
461
+ model.load_state_dict(state_dict, strict=False)
462
+
463
+ return model
464
+
465
+
466
+ if __name__ == "__main__":
467
+
468
+ parser = argparse.ArgumentParser()
469
+ parser.add_argument(
470
+ "checkpoint_dir",
471
+ type=str,
472
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
473
+ parser.add_argument(
474
+ "output_file",
475
+ type=str,
476
+ help=
477
+ "path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)"
478
+ )
479
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
480
+ args = parser.parse_args()
481
+
482
+ debug = args.debug
483
+
484
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file)