debug-internal.log

{"time":"2024-12-16T11:34:20.593727629+08:00","level":"INFO","msg":"using version","core version":"0.19.1"}
{"time":"2024-12-16T11:34:20.593737734+08:00","level":"INFO","msg":"created symlink","path":"/mnt/xgen-mm/LAVIS/wandb/run-20241216_113420-xvp9nqy9/logs/debug-core.log"}
{"time":"2024-12-16T11:34:20.703170654+08:00","level":"INFO","msg":"created new stream","id":"xvp9nqy9"}
{"time":"2024-12-16T11:34:20.703212891+08:00","level":"INFO","msg":"stream: started","id":"xvp9nqy9"}
{"time":"2024-12-16T11:34:20.703229602+08:00","level":"INFO","msg":"handler: started","stream_id":"xvp9nqy9"}
{"time":"2024-12-16T11:34:20.703227695+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"xvp9nqy9"}
{"time":"2024-12-16T11:34:20.70325138+08:00","level":"INFO","msg":"sender: started","stream_id":"xvp9nqy9"}
{"time":"2024-12-16T11:34:30.706322806+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:35753->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T11:34:42.710110122+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:54475->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T11:34:56.93263181+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:34100->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T11:35:16.263212461+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:55712->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T11:35:44.386318183+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:52173->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T11:36:20.624377562+08:00","level":"INFO","msg":"stream: closing","id":"xvp9nqy9"}
{"time":"2024-12-16T11:36:20.624420999+08:00","level":"INFO","msg":"handler: closed","stream_id":"xvp9nqy9"}
{"time":"2024-12-16T11:36:20.624427472+08:00","level":"ERROR","msg":"sender: upsertRun:","error":"failed to upsert bucket: api: failed sending: context canceled"}
{"time":"2024-12-16T11:36:20.624444635+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"xvp9nqy9"}
{"time":"2024-12-16T11:36:20.624466627+08:00","level":"WARN","msg":"runwork: ignoring record after close","work":{"Record":{"RecordType":{"Request":{"RequestType":{"Defer":{}}}},"control":{"always_send":true}}}}
{"time":"2024-12-16T11:36:20.624596137+08:00","level":"INFO","msg":"sender: closed","stream_id":"xvp9nqy9"}
{"time":"2024-12-16T11:36:20.624816752+08:00","level":"INFO","msg":"stream: closed","id":"xvp9nqy9"}