Training in progress, step 2300

checkpoint-1900/latest

@@ -1 +0,0 @@
-global_step1900

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47aebfb81f8363ed05b74e6e078681b273c5c586465aaeef3d06ab2733851a41
+oid sha256:af645f47503053dce382dbd961850c71bc66e93bc686dcdeee4232544a562f2a
 size 10107626487

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2c4b9eee899438e56990c5b63498d9dcb5a31753aaf89888cc5b0f2ae771706
+oid sha256:5ac64f7cbc1d8756d974aad95a2401ba94c38cd165e7ca6bfcf8f0744ae52b25
 size 10107626487

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ea0f41d07ac677f422404956c83f402b623d1e9d38f80d96aa7df1573be8ff3
+oid sha256:1fd21cd79cd151aafba394b3b0a61a3366403fa5c1d45cc4bc474d72d0ee08ad
 size 10107626487

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff2158c21bc75a93ba75193e8cd8dfad392a2398d8707967fcb537e61891b1a3
+oid sha256:5b554f5032c316ccd52d7d0c6e5f68d4eb71efb00f2a285068cbc1285a45936b
 size 10107626487

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c811cf5b1861c8cb662c054b12a7c96a038902adb71f04de6bf96a65b6246ec4
+oid sha256:5aaee965d1cad102275d3f2dc149cc89e9b528caf999974c5a830a30f5439095
 size 10107626487

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9369b4ef2bf8dddb4168a4446f9088e8f3fb3b051aff7a7e19a3049c2070e54
+oid sha256:502586adad5be24d8e8014b38fc72d34964c24d5849f247d6481329d7a6c1201
 size 10107626487

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ef94ff510dd0d525c7d5571350b2b024f9cab3ccce19b0a137485160166c1b8
+oid sha256:1056b37eaa2346f89f6984de8e7ca124e1670381c08ca76458133538f61d9256
 size 10107626487

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa3a389cbb94be741314103205c427d4a3571699efd16e0fd5f242bcb0c9f9f8
+oid sha256:11a9c930edb0fc9d0aa78805da9ff0f6d5c5e62a7bf2daae48f827d4c36166b4
 size 10107626487

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ccec6639f6021228a72cacd939c896abbd5ea2c92885d42b8682b9ec5808e12
+oid sha256:16253da6aa31faba62c481fd127e1e137d42e9bd8a8e0de8bf8ba5249044ce70
 size 168086

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb94590dee0991ff663b4bfea29d11e9fb5258d9f6b09f9dc2c4ed958a22cbbc
+oid sha256:d67f53a8b6d369a2ed5bd600c5a829b97552d13154dd604306e75e47233cfa0b
 size 168086

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd1a137c362ef4d5e939a77f90b7f1238a607611aca192d48e5c14e5590f89c8
+oid sha256:1184d5896d0ea81560a6c9b09767ce2dd7b4265b17d1e1d18a720af9fb803e0d
 size 168086

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00c67c840ba12469a70530ece889168c13d1c236e3c47f561627f2fb998a5811
+oid sha256:f441173b580e67170a4c7a29f843249b2314660419588facecc1232ccbeec5d1
 size 168086

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6cd53ec91a613c2b080addf94707266554f06bea53f22e5f2410638b7c2f359
+oid sha256:25f274add49f75a774044dba3c5da4e33e6fe16fa5cf011200cbf215a95b639a
 size 168086

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8380e0444737b81d5c7bb2a3201de3b50244230313758be4830937fbfcdd62e
+oid sha256:796b632205531f6a09ed1f39ea25566035d7dce6393f75a2a79f00bb7636abdb
 size 168086

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99848a79deee2a4543bbb55739697dc1b1e83d7124d2b9100b203e0aea8ede67
+oid sha256:50cd852c43f022fd55a7e7356b583f5c9311b35f2df5965cfee001b8da212104
 size 168086

{checkpoint-1900/global_step1900 → checkpoint-2300/global_step2300}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c98e12012ad590a146a8cf1c7bce43258a07f093310465ea9b884c26768dfbb4
+oid sha256:a02755b3665b3825d78d37d0f5fb5fdbd2fc668b6ba7aeb1d8a1232ad66b110f
 size 168086

checkpoint-2300/latest

@@ -0,0 +1 @@
+global_step2300

{checkpoint-1900 → checkpoint-2300}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0250158745c111e3a3bd8c0c63bb88e9f24ad6e3b56dea72fd77a43559e57cb
+oid sha256:f4338c6d89388e02f3151e1d3b5b4053fcfcd703725caecf8a1980c3a05490e9
 size 9976576392

{checkpoint-1900 → checkpoint-2300}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf8e87043471e756376e41c4bb6a0d60729025052f4bc413ae38b154f3eaa9d9
+oid sha256:ef4bfeac10f485871643f13bd9d60114f2090397acb8617307a4d91d89bc7d5a
 size 3500296504

{checkpoint-1900 → checkpoint-2300}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6333333333333333,
-  "global_step": 1900,
+  "epoch": 0.7666666666666667,
+  "global_step": 2300,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1146,11 +1146,251 @@
       "learning_rate": 0.0003,
       "loss": 0.0435,
       "step": 1900
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.0412,
+      "step": 1910
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.0441,
+      "step": 1920
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.04,
+      "step": 1930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.041,
+      "step": 1940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.1391,
+      "step": 1950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.1502,
+      "step": 1960
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0767,
+      "step": 1970
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0563,
+      "step": 1980
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0487,
+      "step": 1990
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.042,
+      "step": 2000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.0388,
+      "step": 2010
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.0424,
+      "step": 2020
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.0413,
+      "step": 2030
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.036,
+      "step": 2040
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.0333,
+      "step": 2050
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0334,
+      "step": 2060
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0298,
+      "step": 2070
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0302,
+      "step": 2080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003,
+      "loss": 0.0302,
+      "step": 2090
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003,
+      "loss": 0.0293,
+      "step": 2100
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003,
+      "loss": 0.0286,
+      "step": 2110
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0003,
+      "loss": 0.0279,
+      "step": 2120
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0003,
+      "loss": 0.0258,
+      "step": 2130
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0003,
+      "loss": 0.0259,
+      "step": 2140
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0003,
+      "loss": 0.0273,
+      "step": 2150
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0003,
+      "loss": 0.0253,
+      "step": 2160
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0003,
+      "loss": 0.0247,
+      "step": 2170
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0003,
+      "loss": 0.0237,
+      "step": 2180
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0003,
+      "loss": 0.0231,
+      "step": 2190
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0003,
+      "loss": 0.0213,
+      "step": 2200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0003,
+      "loss": 0.0225,
+      "step": 2210
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0003,
+      "loss": 0.0227,
+      "step": 2220
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0003,
+      "loss": 0.0238,
+      "step": 2230
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0003,
+      "loss": 0.0279,
+      "step": 2240
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0003,
+      "loss": 0.0246,
+      "step": 2250
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0003,
+      "loss": 0.0351,
+      "step": 2260
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0003,
+      "loss": 0.031,
+      "step": 2270
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0003,
+      "loss": 0.0284,
+      "step": 2280
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0003,
+      "loss": 0.0263,
+      "step": 2290
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0003,
+      "loss": 0.0231,
+      "step": 2300
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 795642691584000.0,
+  "total_flos": 963146416128000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a541254ae717180ce583645ab2027b88252e5c9429179141c869f80155d67f8
-size 38691
+oid sha256:39ea659e69e9411156c3ffbe9652606704b81454f5113a213ee14236c2e44f54
+size 40261