[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] optimizer: adam
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] optimizer-delay: 1
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] optimizer-params:
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   []
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] output-omit-bias: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] overwrite: true
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] precision:
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   - float32
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   - float32
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] pretrained-model: ""
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] quantize-biases: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] quantize-bits: 0
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] quantize-log-based: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] quantize-optimization-steps: 0
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] quiet: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] quiet-translation: true
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] relative-paths: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] right-left: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] save-freq: 25
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] seed: 0
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sentencepiece-alphas:
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   - 0.1
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sentencepiece-max-lines: 2000000
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sentencepiece-options: ""
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sharding: local
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] shuffle: batches
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] shuffle-in-ram: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sigterm: save-and-exit
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] skip: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sqlite: ""
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sqlite-drop: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sync-freq: 200u
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] sync-sgd: true
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] tempdir: /home/ubuntu/tasks/task_170500719891218/artifacts/tmp
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] tied-embeddings: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] tied-embeddings-all: True
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] tied-embeddings-src: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] train-embedder-rank:
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   []
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] train-sets:
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   - stdin
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-aan-activation: swish
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-aan-depth: 2
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-aan-nogate: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-decoder-autoreg: self-attention
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-depth-scaling: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-dim-aan: 2048
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-dim-ffn: 2048
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-dropout: 0
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-dropout-attention: 0
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-dropout-ffn: 0
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-ffn-activation: swish
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-ffn-depth: 2
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-guided-alignment-layer: last
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-heads: 8
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-no-projection: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-pool: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-postprocess: dan
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-postprocess-emb: d
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-postprocess-top: ""
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-preprocess: ""
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-tied-layers:
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   []
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] transformer-train-position-embeddings: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] tsv: true
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] tsv-fields: 2
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] type: s2s
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] ulr: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] ulr-dim-emb: 0
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] ulr-dropout: 0
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] ulr-keys-vectors: ""
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] ulr-query-vectors: ""
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] ulr-softmax-temperature: 1
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] ulr-trainable-transformation: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] unlikelihood-loss: false
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] valid-freq: 50
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] valid-log: /home/ubuntu/tasks/task_170500719891218/artifacts/valid.log
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] valid-max-length: 1000
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] valid-metrics:
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   - chrf
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   - ce-mean-words
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config]   - bleu-detok
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] valid-mini-batch: 64
[task 2024-01-11T21:08:11.161Z] [2024-01-11 21:08:11] [config] valid-reset-stalled: true
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] valid-script-args:
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config]   []
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] valid-script-path: ""
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] valid-sets:
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config]   - /home/ubuntu/tasks/task_170500719891218/fetches/devset.enru.tsv
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] valid-translation-output: /home/ubuntu/tasks/task_170500719891218/artifacts/devset.out
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] vocabs:
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config]   - /home/ubuntu/tasks/task_170500719891218/fetches/vocab.spm
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config]   - /home/ubuntu/tasks/task_170500719891218/fetches/vocab.spm
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] word-penalty: 0
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] word-scores: false
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] workspace: 12000
[task 2024-01-11T21:08:11.162Z] [2024-01-11 21:08:11] [config] Model is being created with Marian v1.10.25; e8a1a25 2021-12-07 17:47:33 +0000
[task 2024-01-11T21:08:11.164Z] [2024-01-11 21:08:11] Using synchronous SGD
[task 2024-01-11T21:08:11.164Z] [2024-01-11 21:08:11] [comm] Compiled without MPI support. Running as a single process on translations-1-b-linux-v100-gpu-4-300g-zynlc5-ctm-p4vfnpdp2bg
[task 2024-01-11T21:08:11.164Z] [2024-01-11 21:08:11] Synced seed 1705007291
[task 2024-01-11T21:08:11.164Z] [2024-01-11 21:08:11] [data] Loading SentencePiece vocabulary from file /home/ubuntu/tasks/task_170500719891218/fetches/vocab.spm
[task 2024-01-11T21:08:11.166Z] [2024-01-11 21:08:11] [data] Setting vocabulary size for input 0 to 1,000
[task 2024-01-11T21:08:11.167Z] [2024-01-11 21:08:11] [data] Loading SentencePiece vocabulary from file /home/ubuntu/tasks/task_170500719891218/fetches/vocab.spm
[task 2024-01-11T21:08:11.169Z] [2024-01-11 21:08:11] [data] Setting vocabulary size for input 1 to 1,000
[task 2024-01-11T21:08:11.169Z] [2024-01-11 21:08:11] [batching] Collecting statistics for batch fitting with step size 10
[task 2024-01-11T21:08:13.455Z] [2024-01-11 21:08:13] [memory] Extending reserved space to 12032 MB (device gpu0)
[task 2024-01-11T21:08:13.575Z] [2024-01-11 21:08:13] [memory] Extending reserved space to 12032 MB (device gpu1)
[task 2024-01-11T21:08:13.705Z] [2024-01-11 21:08:13] [memory] Extending reserved space to 12032 MB (device gpu2)
[task 2024-01-11T21:08:13.828Z] [2024-01-11 21:08:13] [memory] Extending reserved space to 12032 MB (device gpu3)
[task 2024-01-11T21:08:13.852Z] [2024-01-11 21:08:13] [comm] Using NCCL 2.8.3 for GPU communication
[task 2024-01-11T21:08:13.852Z] [2024-01-11 21:08:13] [comm] Using global sharding
[task 2024-01-11T21:08:14.274Z] [2024-01-11 21:08:14] [comm] NCCLCommunicators constructed successfully
[task 2024-01-11T21:08:14.275Z] [2024-01-11 21:08:14] [training] Using 4 GPUs
[task 2024-01-11T21:08:14.277Z] [2024-01-11 21:08:14] [logits] Applying loss function for 1 factor(s)
[task 2024-01-11T21:08:14.277Z] [2024-01-11 21:08:14] [memory] Reserving 131 MB, device gpu0
[task 2024-01-11T21:08:14.768Z] [2024-01-11 21:08:14] [gpu] 16-bit TensorCores enabled for float32 matrix operations
[task 2024-01-11T21:08:15.305Z] [2024-01-11 21:08:15] [memory] Reserving 131 MB, device gpu0
[task 2024-01-11T21:09:18.159Z] [2024-01-11 21:09:18] [batching] Done. Typical MB size is 140,468 target words
[task 2024-01-11T21:09:18.317Z] [2024-01-11 21:09:18] [memory] Extending reserved space to 12032 MB (device gpu0)
[task 2024-01-11T21:09:18.588Z] [2024-01-11 21:09:18] [memory] Extending reserved space to 12032 MB (device gpu1)
[task 2024-01-11T21:09:18.644Z] [2024-01-11 21:09:18] [memory] Extending reserved space to 12032 MB (device gpu2)
[task 2024-01-11T21:09:18.676Z] [2024-01-11 21:09:18] [memory] Extending reserved space to 12032 MB (device gpu3)
[task 2024-01-11T21:09:18.709Z] [2024-01-11 21:09:18] [comm] Using NCCL 2.8.3 for GPU communication
[task 2024-01-11T21:09:18.709Z] [2024-01-11 21:09:18] [comm] Using global sharding
[task 2024-01-11T21:09:18.883Z] [2024-01-11 21:09:18] [comm] NCCLCommunicators constructed successfully
[task 2024-01-11T21:09:18.883Z] [2024-01-11 21:09:18] [training] Using 4 GPUs
[task 2024-01-11T21:09:18.884Z] [2024-01-11 21:09:18] Training started
[task 2024-01-11T21:09:19.758Z] [2024-01-11 21:09:19] [Trainer] [INFO] Reading original for epoch 1
[task 2024-01-11T21:09:20.653Z] [2024-01-11 21:09:20] [Trainer] [INFO] Reading original for epoch 2
[task 2024-01-11T21:09:21.530Z] [2024-01-11 21:09:21] [Trainer] [INFO] Reading original for epoch 3
[task 2024-01-11T21:09:22.403Z] [2024-01-11 21:09:22] [Trainer] [INFO] Reading original for epoch 4
[task 2024-01-11T21:09:23.253Z] [2024-01-11 21:09:23] [Trainer] [INFO] Reading original for epoch 5
[task 2024-01-11T21:09:24.137Z] [2024-01-11 21:09:24] [Trainer] [INFO] Reading original for epoch 6
[task 2024-01-11T21:09:25.036Z] [2024-01-11 21:09:25] [Trainer] [INFO] Reading original for epoch 7
[task 2024-01-11T21:09:25.962Z] [2024-01-11 21:09:25] [Trainer] [INFO] Reading original for epoch 8
[task 2024-01-11T21:09:26.832Z] [2024-01-11 21:09:26] [Trainer] [INFO] Reading original for epoch 9
[task 2024-01-11T21:09:27.725Z] [2024-01-11 21:09:27] [Trainer] [INFO] Reading original for epoch 10
[task 2024-01-11T21:09:27.770Z] [2024-01-11 21:09:27] [Trainer] [INFO] waiting for trainer to exit. Press ctrl-c to be more aggressive
[task 2024-01-11T21:09:27.973Z] [2024-01-11 21:09:27] [training] Batches are processed as 1 process(es) x 4 devices/process
[task 2024-01-11T21:09:27.976Z] [2024-01-11 21:09:27] [memory] Reserving 131 MB, device gpu0
[task 2024-01-11T21:09:27.976Z] [2024-01-11 21:09:27] [memory] Reserving 131 MB, device gpu2
[task 2024-01-11T21:09:27.977Z] [2024-01-11 21:09:27] [memory] Reserving 131 MB, device gpu1
[task 2024-01-11T21:09:27.977Z] [2024-01-11 21:09:27] [memory] Reserving 131 MB, device gpu3
[task 2024-01-11T21:09:28.079Z] [2024-01-11 21:09:28] [memory] Reserving 131 MB, device gpu0
[task 2024-01-11T21:09:28.142Z] [2024-01-11 21:09:28] [memory] Reserving 131 MB, device gpu3
[task 2024-01-11T21:09:28.166Z] [2024-01-11 21:09:28] [memory] Reserving 131 MB, device gpu2
[task 2024-01-11T21:09:28.190Z] [2024-01-11 21:09:28] [memory] Reserving 131 MB, device gpu1
[task 2024-01-11T21:09:29.058Z] [2024-01-11 21:09:29] Allocating memory for general optimizer shards
[task 2024-01-11T21:09:29.058Z] [2024-01-11 21:09:29] Parameter type float32, optimization type float32, casting types false
[task 2024-01-11T21:09:29.058Z] [2024-01-11 21:09:29] [memory] Reserving 32 MB, device gpu3
[task 2024-01-11T21:09:29.058Z] [2024-01-11 21:09:29] [memory] Reserving 32 MB, device gpu2
[task 2024-01-11T21:09:29.058Z] [2024-01-11 21:09:29] Parameter type float32, optimization type float32, casting types false
[task 2024-01-11T21:09:29.058Z] [2024-01-11 21:09:29] [memory] Reserving 32 MB, device gpu1
[task 2024-01-11T21:09:29.058Z] [2024-01-11 21:09:29] [memory] Reserving 32 MB, device gpu0
[task 2024-01-11T21:09:29.065Z] [2024-01-11 21:09:29] Allocating memory for Adam-specific shards
[task 2024-01-11T21:09:29.065Z] [2024-01-11 21:09:29] [memory] Reserving 65 MB, device gpu3
[task 2024-01-11T21:09:29.065Z] [2024-01-11 21:09:29] [memory] Reserving 65 MB, device gpu2
[task 2024-01-11T21:09:29.065Z] [2024-01-11 21:09:29] [memory] Reserving 65 MB, device gpu0
[task 2024-01-11T21:09:29.065Z] [2024-01-11 21:09:29] [memory] Reserving 65 MB, device gpu1
[task 2024-01-11T21:09:29.070Z] [2024-01-11 21:09:29] Ep. 1 : Up. 1 : Sen. 1,756 : Cost 7.24135208 : Time 10.79s : 4600.43 words/s : gNorm 2.3186
[task 2024-01-11T21:09:29.457Z] [2024-01-11 21:09:29] Ep. 1 : Up. 2 : Sen. 3,764 : Cost 6.83197355 : Time 0.39s : 114850.68 words/s : gNorm 2.1777
[task 2024-01-11T21:09:30.380Z] [2024-01-11 21:09:30] Ep. 1 : Up. 3 : Sen. 5,168 : Cost 6.62698460 : Time 0.92s : 120412.83 words/s : gNorm 2.1433
[task 2024-01-11T21:09:31.206Z] [2024-01-11 21:09:31] Ep. 1 : Up. 4 : Sen. 6,572 : Cost 6.39289808 : Time 0.83s : 117583.63 words/s : gNorm 2.0266
[task 2024-01-11T21:09:31.714Z] [2024-01-11 21:09:31] Ep. 1 : Up. 5 : Sen. 8,132 : Cost 6.21113586 : Time 0.51s : 105373.70 words/s : gNorm 1.7984
[task 2024-01-11T21:09:31.746Z] [2024-01-11 21:09:31] Ep. 1 : Up. 6 : Sen. 8,186 : Cost 6.10178709 : Time 0.03s : 9024.36 words/s : gNorm 3.6183
[task 2024-01-11T21:09:32.813Z] [2024-01-11 21:09:32] Ep. 1 : Up. 7 : Sen. 9,590 : Cost 5.99499273 : Time 1.07s : 122454.93 words/s : gNorm 3.2057
[task 2024-01-11T21:09:33.252Z] [2024-01-11 21:09:33] Ep. 1 : Up. 8 : Sen. 11,346 : Cost 5.93519115 : Time 0.44s : 102675.51 words/s : gNorm 2.7727
[task 2024-01-11T21:09:33.789Z] [2024-01-11 21:09:33] Ep. 1 : Up. 9 : Sen. 12,906 : Cost 5.92996168 : Time 0.54s : 111053.70 words/s : gNorm 2.4353
[task 2024-01-11T21:09:34.750Z] [2024-01-11 21:09:34] Ep. 1 : Up. 10 : Sen. 14,310 : Cost 5.75753450 : Time 0.96s : 123402.44 words/s : gNorm 2.1995
[task 2024-01-11T21:09:35.354Z] [2024-01-11 21:09:35] Ep. 1 : Up. 11 : Sen. 15,714 : Cost 5.83631229 : Time 0.60s : 103968.28 words/s : gNorm 1.9910
[task 2024-01-11T21:09:35.981Z] [2024-01-11 21:09:35] Ep. 1 : Up. 12 : Sen. 17,118 : Cost 5.74326038 : Time 0.63s : 103689.94 words/s : gNorm 1.8256
[task 2024-01-11T21:09:36.857Z] [2024-01-11 21:09:36] Ep. 1 : Up. 13 : Sen. 18,522 : Cost 5.64789486 : Time 0.88s : 120857.27 words/s : gNorm 1.6921
[task 2024-01-11T21:09:37.404Z] [2024-01-11 21:09:37] Ep. 1 : Up. 14 : Sen. 20,082 : Cost 5.62230873 : Time 0.55s : 103771.43 words/s : gNorm 1.5853
[task 2024-01-11T21:09:38.144Z] [2024-01-11 21:09:38] Ep. 1 : Up. 15 : Sen. 21,486 : Cost 5.56740046 : Time 0.74s : 113584.72 words/s : gNorm 1.4931
[task 2024-01-11T21:09:38.866Z] [2024-01-11 21:09:38] Ep. 1 : Up. 16 : Sen. 22,890 : Cost 5.52759171 : Time 0.72s : 112641.07 words/s : gNorm 1.4142
[task 2024-01-11T21:09:39.574Z] [2024-01-11 21:09:39] Ep. 1 : Up. 17 : Sen. 24,294 : Cost 5.50164604 : Time 0.71s : 111120.29 words/s : gNorm 1.3420
[task 2024-01-11T21:09:40.160Z] [2024-01-11 21:09:40] Ep. 1 : Up. 18 : Sen. 25,698 : Cost 5.44184351 : Time 0.59s : 103734.23 words/s : gNorm 1.2835
[task 2024-01-11T21:09:40.828Z] [2024-01-11 21:09:40] Ep. 1 : Up. 19 : Sen. 27,102 : Cost 5.41650200 : Time 0.67s : 107381.39 words/s : gNorm 1.2319
[task 2024-01-11T21:09:41.611Z] [2024-01-11 21:09:41] Ep. 1 : Up. 20 : Sen. 28,506 : Cost 5.36061192 : Time 0.78s : 115230.24 words/s : gNorm 1.1848
[task 2024-01-11T21:09:42.114Z] [2024-01-11 21:09:42] Ep. 1 : Up. 21 : Sen. 30,066 : Cost 5.25943232 : Time 0.50s : 100741.09 words/s : gNorm 1.1545
[task 2024-01-11T21:09:42.751Z] [2024-01-11 21:09:42] Ep. 1 : Up. 22 : Sen. 31,470 : Cost 5.27156115 : Time 0.64s : 105152.95 words/s : gNorm 1.1222
[task 2024-01-11T21:09:43.203Z] [2024-01-11 21:09:43] Ep. 1 : Up. 23 : Sen. 33,226 : Cost 5.16485834 : Time 0.45s : 118277.80 words/s : gNorm 1.1027
[task 2024-01-11T21:09:43.970Z] [2024-01-11 21:09:43] Ep. 1 : Up. 24 : Sen. 34,630 : Cost 5.19723511 : Time 0.77s : 113635.53 words/s : gNorm 1.0774
[task 2024-01-11T21:09:44.821Z] [2024-01-11 21:09:44] Ep. 1 : Up. 25 : Sen. 36,034 : Cost 5.10784483 : Time 0.85s : 119296.37 words/s : gNorm 1.0521
[task 2024-01-11T21:09:44.825Z] [2024-01-11 21:09:44] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz
[task 2024-01-11T21:09:46.082Z] [2024-01-11 21:09:46] Saving Adam parameters
[task 2024-01-11T21:09:48.342Z] [2024-01-11 21:09:48] [training] Saving training checkpoint to /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz and /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.optimizer.npz
[task 2024-01-11T21:09:50.638Z] [2024-01-11 21:09:50] Ep. 1 : Up. 26 : Sen. 37,438 : Cost 5.13091326 : Time 5.82s : 12716.48 words/s : gNorm 1.0284
[task 2024-01-11T21:09:51.208Z] [2024-01-11 21:09:51] Ep. 1 : Up. 27 : Sen. 38,842 : Cost 5.04033232 : Time 0.57s : 98959.30 words/s : gNorm 1.0074
[task 2024-01-11T21:09:51.894Z] [2024-01-11 21:09:51] Ep. 1 : Up. 28 : Sen. 40,246 : Cost 4.99159908 : Time 0.69s : 111272.00 words/s : gNorm 0.9842
[task 2024-01-11T21:09:52.242Z] [2024-01-11 21:09:52] Ep. 1 : Up. 29 : Sen. 42,590 : Cost 4.84732676 : Time 0.35s : 106972.31 words/s : gNorm 1.0150
[task 2024-01-11T21:09:52.893Z] [2024-01-11 21:09:52] Ep. 1 : Up. 30 : Sen. 43,994 : Cost 4.93669319 : Time 0.65s : 106826.35 words/s : gNorm 1.0018
[task 2024-01-11T21:09:53.690Z] [2024-01-11 21:09:53] Ep. 1 : Up. 31 : Sen. 45,398 : Cost 4.86935568 : Time 0.80s : 117297.15 words/s : gNorm 0.9853
[task 2024-01-11T21:09:54.271Z] [2024-01-11 21:09:54] Ep. 1 : Up. 32 : Sen. 46,802 : Cost 4.87597990 : Time 0.58s : 100539.12 words/s : gNorm 0.9692
[task 2024-01-11T21:09:54.273Z] [2024-01-11 21:09:54] Seen 46,802 samples
[task 2024-01-11T21:09:54.274Z] [2024-01-11 21:09:54] Starting data epoch 2 in logical epoch 2
[task 2024-01-11T21:09:54.274Z] [2024-01-11 21:09:54] Training finished
[task 2024-01-11T21:09:55.588Z] [2024-01-11 21:09:55] [valid] First sentence's tokens as scored:
[task 2024-01-11T21:09:55.588Z] [2024-01-11 21:09:55] [valid] Decoding validation set with SentencePieceVocab for scoring
[task 2024-01-11T21:09:55.588Z] [2024-01-11 21:09:55] [valid]   Hyp: В ы .
[task 2024-01-11T21:09:55.589Z] [2024-01-11 21:09:55] [valid]   Ref: У п о б е д и т е л е й в с т р е ч и г о л а м и о т м е т и л и с ь Л а у т а р о М а р т и н е с и М а т т е о П о л и т а н о , з а б и в ш и е н а 1 2 - й и 8 9 - й м и н у т а х с о о т в е т с т в е н н о .
[task 2024-01-11T21:10:13.553Z] [2024-01-11 21:10:13] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-chrf.npz
[task 2024-01-11T21:10:14.114Z] [2024-01-11 21:10:14] [valid] Ep. 2 : Up. 32 : chrf : 0.758845 : new best
[task 2024-01-11T21:10:15.282Z] [2024-01-11 21:10:15] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-ce-mean-words.npz
[task 2024-01-11T21:10:15.830Z] [2024-01-11 21:10:15] [valid] Ep. 2 : Up. 32 : ce-mean-words : 5.10715 : new best
[task 2024-01-11T21:10:34.639Z] [2024-01-11 21:10:34] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-bleu-detok.npz
[task 2024-01-11T21:10:35.222Z] [2024-01-11 21:10:35] [valid] Ep. 2 : Up. 32 : bleu-detok : 0 : new best
[task 2024-01-11T21:10:35.222Z] [2024-01-11 21:10:35] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz
[task 2024-01-11T21:10:38.475Z] [2024-01-11 21:10:38] Saving Adam parameters
[task 2024-01-11T21:10:40.608Z] [2024-01-11 21:10:40] [training] Saving training checkpoint to /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz and /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.optimizer.npz
[task 2024-01-11T21:10:49.210Z] + cp /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-chrf.npz /home/ubuntu/tasks/task_170500719891218/artifacts/final.model.npz.best-chrf.npz
[task 2024-01-11T21:10:49.350Z] + cp /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-chrf.npz.decoder.yml /home/ubuntu/tasks/task_170500719891218/artifacts/final.model.npz.best-chrf.npz.decoder.yml
[task 2024-01-11T21:10:49.352Z] + echo '### Model training is completed: /home/ubuntu/tasks/task_170500719891218/artifacts'
[task 2024-01-11T21:10:49.352Z] ### Model training is completed: /home/ubuntu/tasks/task_170500719891218/artifacts
[task 2024-01-11T21:10:49.352Z] + echo '###### Done: Training a model'
[task 2024-01-11T21:10:49.352Z] ###### Done: Training a model
[task 2024-01-11T21:10:49.353Z] + '[' None == None ']'
[task 2024-01-11T21:10:49.353Z] + cp /home/ubuntu/tasks/task_170500719891218/fetches/vocab.spm /home/ubuntu/tasks/task_170500719891218/artifacts
[fetches 2024-01-11T21:10:49.356Z] removing /home/ubuntu/tasks/task_170500719891218/fetches
[fetches 2024-01-11T21:10:49.713Z] finished
[taskcluster 2024-01-11T21:10:49.724Z]    Exit Code: 0
[taskcluster 2024-01-11T21:10:49.724Z]    User Time: 5m33.567375s
[taskcluster 2024-01-11T21:10:49.724Z]  Kernel Time: 23.165214s
[taskcluster 2024-01-11T21:10:49.724Z]    Wall Time: 4m8.4116062s
[taskcluster 2024-01-11T21:10:49.724Z]       Result: SUCCEEDED
[taskcluster 2024-01-11T21:10:49.724Z] === Task Finished ===
[taskcluster 2024-01-11T21:10:49.724Z] Task Duration: 4m8.432432896s
[taskcluster 2024-01-11T21:11:01.024Z] Uploading artifact public/build/config.opustrainer.yml from file /home/ubuntu/tasks/task_170500719891218/artifacts/config.opustrainer.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:01.228Z] Uploading artifact public/build/config.opustrainer.yml.state from file /home/ubuntu/tasks/task_170500719891218/artifacts/config.opustrainer.yml.state with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:01.447Z] Uploading artifact public/build/devset.out from file /home/ubuntu/tasks/task_170500719891218/artifacts/devset.out with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:01.635Z] Uploading artifact public/build/final.model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_170500719891218/artifacts/final.model.npz.best-chrf.npz with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:08.844Z] Uploading artifact public/build/final.model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_170500719891218/artifacts/final.model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:09.007Z] Uploading artifact public/build/model.npz from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:16.586Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-bleu-detok.npz with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:23.350Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz.decoder.yml from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-bleu-detok.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:23.609Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-ce-mean-words.npz with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:30.288Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz.decoder.yml from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-ce-mean-words.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:30.456Z] Uploading artifact public/build/model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-chrf.npz with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:37.493Z] Uploading artifact public/build/model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:37.706Z] Uploading artifact public/build/model.npz.decoder.yml from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:11:37.940Z] Uploading artifact public/build/model.npz.optimizer.npz from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.optimizer.npz with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:12:04.556Z] Uploading artifact public/build/model.npz.progress.yml from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.progress.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:12:04.732Z] Uploading artifact public/build/model.npz.yml from file /home/ubuntu/tasks/task_170500719891218/artifacts/model.npz.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:12:04.904Z] Uploading artifact public/build/opustrainer.log from file /home/ubuntu/tasks/task_170500719891218/artifacts/opustrainer.log with content encoding "gzip", mime type "text/plain" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:12:05.071Z] Uploading artifact public/build/train.log from file /home/ubuntu/tasks/task_170500719891218/artifacts/train.log with content encoding "gzip", mime type "text/plain" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:12:05.239Z] Uploading artifact public/build/valid.log from file /home/ubuntu/tasks/task_170500719891218/artifacts/valid.log with content encoding "gzip", mime type "text/plain" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:12:05.409Z] Uploading artifact public/build/vocab.spm from file /home/ubuntu/tasks/task_170500719891218/artifacts/vocab.spm with content encoding "gzip", mime type "application/x-source-rpm" and expiry 2024-04-10T20:54:27.597Z
[taskcluster 2024-01-11T21:12:05.574Z] [mounts] Preserving cache: Moving "/home/ubuntu/tasks/task_170500719891218/checkouts" to "/home/ubuntu/caches/YL9rb4WKRUGJAZ7ef8n0Zw"
[taskcluster 2024-01-11T21:12:05.654Z] Uploading link artifact public/logs/live.log to artifact public/logs/live_backing.log with expiry 2024-04-10T20:54:27.597Z