debug-internal.log

{"time":"2024-12-16T10:16:27.456947987+08:00","level":"INFO","msg":"using version","core version":"0.19.1"}
{"time":"2024-12-16T10:16:27.456957573+08:00","level":"INFO","msg":"created symlink","path":"/mnt/xgen-mm/LAVIS/wandb/run-20241216_101627-3las4gzs/logs/debug-core.log"}
{"time":"2024-12-16T10:16:27.567756055+08:00","level":"INFO","msg":"created new stream","id":"3las4gzs"}
{"time":"2024-12-16T10:16:27.567795709+08:00","level":"INFO","msg":"stream: started","id":"3las4gzs"}
{"time":"2024-12-16T10:16:27.567812708+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"3las4gzs"}
{"time":"2024-12-16T10:16:27.567813267+08:00","level":"INFO","msg":"sender: started","stream_id":"3las4gzs"}
{"time":"2024-12-16T10:16:27.567816566+08:00","level":"INFO","msg":"handler: started","stream_id":"3las4gzs"}
{"time":"2024-12-16T10:16:37.570095655+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:33868->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:16:50.053546304+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:51552->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:17:04.464940038+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:37031->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:17:23.179941866+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:45848->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:17:49.704248141+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 114.114.114.114:53: read udp 10.16.4.6:58012->114.114.114.114:53: i/o timeout"}
{"time":"2024-12-16T10:17:57.499829779+08:00","level":"INFO","msg":"stream: closing","id":"3las4gzs"}
{"time":"2024-12-16T10:17:57.499987602+08:00","level":"INFO","msg":"handler: closed","stream_id":"3las4gzs"}
{"time":"2024-12-16T10:17:57.500002408+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"3las4gzs"}
{"time":"2024-12-16T10:17:57.500022511+08:00","level":"ERROR","msg":"sender: upsertRun:","error":"failed to upsert bucket: api: failed sending: context canceled"}
{"time":"2024-12-16T10:17:57.500065863+08:00","level":"WARN","msg":"runwork: ignoring record after close","work":{"Record":{"RecordType":{"Request":{"RequestType":{"Defer":{}}}},"control":{"always_send":true}}}}
{"time":"2024-12-16T10:17:57.500185254+08:00","level":"INFO","msg":"sender: closed","stream_id":"3las4gzs"}
{"time":"2024-12-16T10:17:57.500386839+08:00","level":"INFO","msg":"stream: closed","id":"3las4gzs"}