pszemraj commited on
Commit
89ac41d
1 Parent(s): 665558b

Upload ds_config_zero2.json

Browse files
Files changed (1) hide show
  1. ds_config_zero2.json +39 -0
ds_config_zero2.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "amp": {
3
+ "enabled": "auto",
4
+ "opt_level": "auto"
5
+ },
6
+
7
+ "optimizer": {
8
+ "type": "AdamW",
9
+ "params": {
10
+ "lr": "auto",
11
+ "betas": "auto",
12
+ "eps": "auto",
13
+ "weight_decay": "auto"
14
+ }
15
+ },
16
+
17
+
18
+ "zero_optimization": {
19
+ "stage": 2,
20
+ "offload_optimizer": {
21
+ "device": "cpu",
22
+ "pin_memory": true
23
+ },
24
+ "allgather_partitions": true,
25
+ "allgather_bucket_size": 2e8,
26
+ "overlap_comm": true,
27
+ "reduce_scatter": true,
28
+ "reduce_bucket_size": 2e8,
29
+ "round_robin_gradients": true,
30
+ "contiguous_gradients": true
31
+ },
32
+
33
+ "gradient_accumulation_steps": "auto",
34
+ "gradient_clipping": "auto",
35
+ "steps_per_print": 4000,
36
+ "train_batch_size": "auto",
37
+ "train_micro_batch_size_per_gpu": "auto",
38
+ "wall_clock_breakdown": false
39
+ }