debug-internal.log

{"time":"2024-12-16T10:48:34.438350079+08:00","level":"INFO","msg":"using version","core version":"0.19.1"}
{"time":"2024-12-16T10:48:34.438360599+08:00","level":"INFO","msg":"created symlink","path":"/mnt/xgen-mm/LAVIS/wandb/run-20241216_104834-iey8t0re/logs/debug-core.log"}
{"time":"2024-12-16T10:48:34.548423887+08:00","level":"INFO","msg":"created new stream","id":"iey8t0re"}
{"time":"2024-12-16T10:48:34.548468636+08:00","level":"INFO","msg":"stream: started","id":"iey8t0re"}
{"time":"2024-12-16T10:48:34.548487791+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"iey8t0re"}
{"time":"2024-12-16T10:48:34.548495322+08:00","level":"INFO","msg":"sender: started","stream_id":"iey8t0re"}
{"time":"2024-12-16T10:48:34.548523119+08:00","level":"INFO","msg":"handler: started","stream_id":"iey8t0re"}
{"time":"2024-12-16T10:48:44.551383213+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:37994->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:48:56.609144292+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:36384->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:49:10.850134383+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:45542->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:49:28.967945371+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:43370->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:49:57.438447961+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:33213->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:50:04.461575809+08:00","level":"INFO","msg":"stream: closing","id":"iey8t0re"}
{"time":"2024-12-16T10:50:04.461655516+08:00","level":"ERROR","msg":"sender: upsertRun:","error":"failed to upsert bucket: api: failed sending: context canceled"}
{"time":"2024-12-16T10:50:04.461679932+08:00","level":"INFO","msg":"handler: closed","stream_id":"iey8t0re"}
{"time":"2024-12-16T10:50:04.461690188+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"iey8t0re"}
{"time":"2024-12-16T10:50:04.461739964+08:00","level":"WARN","msg":"runwork: ignoring record after close","work":{"Record":{"RecordType":{"Request":{"RequestType":{"Defer":{}}}},"control":{"always_send":true}}}}
{"time":"2024-12-16T10:50:04.461870979+08:00","level":"INFO","msg":"sender: closed","stream_id":"iey8t0re"}
{"time":"2024-12-16T10:50:04.462185417+08:00","level":"INFO","msg":"stream: closed","id":"iey8t0re"}