{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.825:637801): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.825:637801): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=8525517 dev=00:ae mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.825:637801): item=0 name=\"/usr/bin/cat\" inode=8524565 dev=00:ae mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.825:637801): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.825:637801): argc=2 a0=\"cat\" a1=\"/sys/fs/cgroup//memory.stat\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.825:637801): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.825:637801): arch=c000003e syscall=59 success=yes exit=0 a0=7b6e3fe43d60 a1=7b6e1d663c00 a2=617d5cb4ea40 a3=8 items=2 ppid=5585 pid=1823499 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"cat\" exe=\"/usr/bin/cat\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.825:637800): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.825:637800): item=0 name=\"/usr/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.825:637800): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.825:637800): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663c00 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823499 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.825:637799): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.825:637799): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.825:637799): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.825:637799): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663c00 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823499 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.825:637798): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.825:637798): item=0 name=\"/usr/local/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.825:637798): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.825:637798): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663c00 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823499 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.825:637797): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.825:637797): item=0 name=\"/tmp/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.825:637797): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.825:637797): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663c00 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823499 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.825:637796): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.825:637796): item=0 name=\"/root/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.825:637796): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.825:637796): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663c00 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823499 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.825:637795): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.825:637795): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.825:637795): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.825:637795): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663c00 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823499 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.822:637794): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.822:637794): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=8525517 dev=00:ae mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.822:637794): item=0 name=\"/usr/bin/cat\" inode=8524565 dev=00:ae mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.822:637794): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.822:637794): argc=2 a0=\"cat\" a1=\"/sys/fs/cgroup//memory.current\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.822:637794): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.822:637794): arch=c000003e syscall=59 success=yes exit=0 a0=7b6e3fe43d60 a1=7b6e1d663be0 a2=617d5cb4ea40 a3=8 items=2 ppid=5585 pid=1823498 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"cat\" exe=\"/usr/bin/cat\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.822:637793): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.822:637793): item=0 name=\"/usr/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.822:637793): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.822:637793): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663be0 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823498 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.822:637792): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.822:637792): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.822:637792): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.822:637792): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663be0 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823498 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.822:637791): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.822:637791): item=0 name=\"/usr/local/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.822:637791): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.822:637791): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663be0 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823498 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.822:637790): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.822:637790): item=0 name=\"/tmp/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.822:637790): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.822:637790): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663be0 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823498 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.822:637789): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.822:637789): item=0 name=\"/root/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.822:637789): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.822:637789): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663be0 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823498 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.822:637788): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.822:637788): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.822:637788): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.822:637788): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6e1d663be0 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823498 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.819:637787): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.819:637787): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=8525517 dev=00:ae mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.819:637787): item=0 name=\"/usr/bin/cat\" inode=8524565 dev=00:ae mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.819:637787): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.819:637787): argc=2 a0=\"cat\" a1=\"/proc/self/cgroup\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.819:637787): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.819:637787): arch=c000003e syscall=59 success=yes exit=0 a0=7b6e3fe43d60 a1=7b6dcb20af60 a2=617d5cb4ea40 a3=8 items=2 ppid=5585 pid=1823497 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"cat\" exe=\"/usr/bin/cat\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.819:637786): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.819:637786): item=0 name=\"/usr/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.819:637786): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.819:637786): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6dcb20af60 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823497 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.819:637785): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.819:637785): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.819:637785): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.819:637785): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6dcb20af60 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823497 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.818:637784): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.818:637784): item=0 name=\"/usr/local/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.818:637784): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.818:637784): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6dcb20af60 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823497 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.818:637783): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.818:637783): item=0 name=\"/tmp/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.818:637783): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.818:637783): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6dcb20af60 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823497 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.818:637782): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.818:637782): item=0 name=\"/root/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.818:637782): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.818:637782): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6dcb20af60 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823497 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.818:637781): proctitle=77696E646D696C6C00736572766572"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.818:637781): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.818:637781): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.818:637781): arch=c000003e syscall=59 success=no exit=-2 a0=7b6e3fe43d60 a1=7b6dcb20af60 a2=617d5cb4ea40 a3=8 items=1 ppid=5585 pid=1823497 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.529:637780): proctitle=6375726C002D6600687474703A2F2F6C6F63616C686F73743A383030302F6170692F76312F686561727462656174"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.529:637780): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=6817536 dev=00:a6 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.529:637780): item=0 name=\"/usr/bin/curl\" inode=6830784 dev=00:a6 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.529:637780): cwd=\"/chroma\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.529:637780): argc=3 a0=\"curl\" a1=\"-f\" a2=\"http://localhost:8000/api/v1/heartbeat\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.529:637780): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.529:637780): arch=c000003e syscall=59 success=yes exit=0 a0=c00018aeb0 a1=c000022ac0 a2=c000134930 a3=0 items=2 ppid=1823477 pid=1823490 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"curl\" exe=\"/usr/bin/curl\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.487:637779): proctitle=72756E6300696E6974"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.487:637779): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=4108210 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.487:637779): item=0 name=\"/proc/self/fd/6\" inode=4092559 dev=00:bb mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.487:637779): cwd=\"/var/lib/docker/rootfs/overlayfs/15d7c01eab750dd5d1697f885100eb074f3a5095b1e73b1cc7a531eedb613268\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.487:637779): argc=2 a0=\"runc\" a1=\"init\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.487:637779): arch=c000003e syscall=59 success=yes exit=0 a0=c000245890 a1=c000249350 a2=c00024bc80 a3=0 items=2 ppid=1823477 pid=1823486 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"6\" exe=\"/runc\" subj=unconfined key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.472:637778): proctitle=72756E63002D2D726F6F74002F7661722F72756E2F646F636B65722F72756E74696D652D72756E632F6D6F6279002D2D6C6F67002F72756E2F636F6E7461696E6572642F696F2E636F6E7461696E6572642E72756E74696D652E76322E7461736B2F6D6F62792F31356437633031656162373530646435643136393766383835"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.472:637778): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=4108210 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.472:637778): item=0 name=\"/usr/bin/runc\" inode=4092559 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.472:637778): cwd=\"/run/containerd/io.containerd.runtime.v2.task/moby/15d7c01eab750dd5d1697f885100eb074f3a5095b1e73b1cc7a531eedb613268\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.472:637778): argc=15 a0=\"runc\" a1=\"--root\" a2=\"/var/run/docker/runtime-runc/moby\" a3=\"--log\" a4=\"/run/containerd/io.containerd.runtime.v2.task/moby/15d7c01eab750dd5d1697f885100eb074f3a5095b1e73b1cc7a531eedb613268/log.json\" a5=\"--log-format\" a6=\"json\" a7=\"--systemd-cgroup\" a8=\"exec\" a9=\"--process\" a10=\"/tmp/runc-process4256636493\" a11=\"--detach\" a12=\"--pid-file\" a13=\"/run/containerd/io.containerd.runtime.v2.task/moby/15d7c01eab750dd5d1697f885100eb074f3a5095b1e73b1cc7a531eedb613268/b6389c54b6ccc1beb984105ad475e10bebebe508a5a9161767e3abad1713635c.pid\" a14=\"15d7c01eab750dd5d1697f885100eb074f3a5095b1e73b1cc7a531eedb613268\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.472:637778): arch=c000003e syscall=59 success=yes exit=0 a0=c0002c03e0 a1=c0001d6a00 a2=c0001d6a80 a3=0 items=2 ppid=4342 pid=1823477 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"runc\" exe=\"/usr/bin/runc\" subj=unconfined key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.190:637777): proctitle=6D617269616462002D2D64656661756C74732D65787472612D66696C653D2F7661722F6C69622F6D7973716C2F2E6D792D6865616C7468636865636B2E636E66002D2D70726F746F636F6C00736F636B6574002D42002D2D736B69702D636F6C756D6E2D6E616D6573002D650073656C65637420312066726F6D20696E666F72"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.190:637777): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=7089648 dev=00:a2 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.190:637777): item=0 name=\"/usr/bin/mariadb\" inode=7117817 dev=00:a2 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.190:637777): cwd=\"/\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.190:637777): argc=8 a0=\"mariadb\" a1=\"--defaults-extra-file=/var/lib/mysql/.my-healthcheck.cnf\" a2=\"--protocol\" a3=\"socket\" a4=\"-B\" a5=\"--skip-column-names\" a6=\"-e\" a7=73656C65637420312066726F6D20696E666F726D6174696F6E5F736368656D612E454E47494E455320574845524520656E67696E653D27696E6E6F64622720414E4420737570706F727420696E202827594553272C202744454641554C54272C2027454E41424C45442729"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.190:637777): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.190:637777): arch=c000003e syscall=59 success=yes exit=0 a0=5683f9c8c990 a1=5683f9cae9c0 a2=5683f9c8d860 a3=8 items=2 ppid=1823448 pid=1823475 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"mariadb\" exe=\"/usr/bin/mariadb\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.172:637776): proctitle=72756E6300696E6974"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.172:637776): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=4108210 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.172:637776): item=0 name=\"/proc/self/fd/6\" inode=4092559 dev=00:bb mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.172:637776): cwd=\"/var/lib/docker/rootfs/overlayfs/531ab8968ce844b18b0b3ebaf9e6e80c7b8f7cc1974293e5f9eb3ac90eba0e50\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.172:637776): argc=2 a0=\"runc\" a1=\"init\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.172:637776): arch=c000003e syscall=59 success=yes exit=0 a0=c00017b860 a1=c00017f338 a2=c000181c40 a3=0 items=2 ppid=1823454 pid=1823466 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"6\" exe=\"/runc\" subj=unconfined key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.170:637775): proctitle=6D617269616462002D2D64656661756C74732D65787472612D66696C653D2F7661722F6C69622F6D7973716C2F2E6D792D6865616C7468636865636B2E636E66002D2D70726F746F636F6C00736F636B6574002D42002D2D736B69702D636F6C756D6E2D6E616D6573002D650073656C656374204040736B69705F6E6574776F"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.170:637775): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=7089648 dev=00:a2 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.170:637775): item=0 name=\"/usr/bin/mariadb\" inode=7117817 dev=00:a2 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.170:637775): cwd=\"/\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.170:637775): argc=8 a0=\"mariadb\" a1=\"--defaults-extra-file=/var/lib/mysql/.my-healthcheck.cnf\" a2=\"--protocol\" a3=\"socket\" a4=\"-B\" a5=\"--skip-column-names\" a6=\"-e\" a7=73656C656374204040736B69705F6E6574776F726B696E67"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.170:637775): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.170:637775): arch=c000003e syscall=59 success=yes exit=0 a0=5683f9c8cfe0 a1=5683f9cae810 a2=5683f9c8d4a0 a3=8 items=2 ppid=1823448 pid=1823460 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"mariadb\" exe=\"/usr/bin/mariadb\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.161:637774): proctitle=2F62696E2F62617368002F7573722F6C6F63616C2F62696E2F6865616C7468636865636B2E7368002D2D636F6E6E656374002D2D696E6E6F64625F696E697469616C697A6564"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.161:637774): item=2 name=\"/lib64/ld-linux-x86-64.so.2\" inode=7089648 dev=00:a2 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.161:637774): item=1 name=\"/bin/bash\" inode=6963796 dev=00:a2 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.161:637774): item=0 name=\"/usr/local/bin/healthcheck.sh\" inode=7348502 dev=00:a2 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.161:637774): cwd=\"/\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.161:637774): argc=4 a0=\"/bin/bash\" a1=\"/usr/local/bin/healthcheck.sh\" a2=\"--connect\" a3=\"--innodb_initialized\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.161:637774): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.161:637774): arch=c000003e syscall=59 success=yes exit=0 a0=c000028760 a1=c000022660 a2=c0000de320 a3=0 items=3 ppid=1823432 pid=1823448 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"healthcheck.sh\" exe=\"/usr/bin/bash\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.155:637773): proctitle=72756E63002D2D726F6F74002F7661722F72756E2F646F636B65722F72756E74696D652D72756E632F6D6F6279002D2D6C6F67002F72756E2F636F6E7461696E6572642F696F2E636F6E7461696E6572642E72756E74696D652E76322E7461736B2F6D6F62792F35333161623839363863653834346231386230623365626166"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.155:637773): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=4108210 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.155:637773): item=0 name=\"/usr/bin/runc\" inode=4092559 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.155:637773): cwd=\"/run/containerd/io.containerd.runtime.v2.task/moby/531ab8968ce844b18b0b3ebaf9e6e80c7b8f7cc1974293e5f9eb3ac90eba0e50\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.155:637773): argc=15 a0=\"runc\" a1=\"--root\" a2=\"/var/run/docker/runtime-runc/moby\" a3=\"--log\" a4=\"/run/containerd/io.containerd.runtime.v2.task/moby/531ab8968ce844b18b0b3ebaf9e6e80c7b8f7cc1974293e5f9eb3ac90eba0e50/log.json\" a5=\"--log-format\" a6=\"json\" a7=\"--systemd-cgroup\" a8=\"exec\" a9=\"--process\" a10=\"/tmp/runc-process2191776381\" a11=\"--detach\" a12=\"--pid-file\" a13=\"/run/containerd/io.containerd.runtime.v2.task/moby/531ab8968ce844b18b0b3ebaf9e6e80c7b8f7cc1974293e5f9eb3ac90eba0e50/89a63d8056a2f6ca02557765fcac633a491998acb7a4e06816a22949b603cd79.pid\" a14=\"531ab8968ce844b18b0b3ebaf9e6e80c7b8f7cc1974293e5f9eb3ac90eba0e50\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.155:637773): arch=c000003e syscall=59 success=yes exit=0 a0=c000104720 a1=c00020c880 a2=c00020c900 a3=0 items=2 ppid=4133 pid=1823454 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"runc\" exe=\"/usr/bin/runc\" subj=unconfined key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.114:637772): proctitle=72756E6300696E6974"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.114:637772): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=4108210 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.114:637772): item=0 name=\"/proc/self/fd/6\" inode=4092559 dev=00:bb mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.114:637772): cwd=\"/var/lib/docker/rootfs/overlayfs/7ede019e3cb0e889018cab8bf0c09e5e0f9742f2ec242b172ed023e182cb56d7\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.114:637772): argc=2 a0=\"runc\" a1=\"init\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.114:637772): arch=c000003e syscall=59 success=yes exit=0 a0=c0001eb840 a1=c0001ef338 a2=c0001f1c40 a3=0 items=2 ppid=1823432 pid=1823444 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"6\" exe=\"/runc\" subj=unconfined key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.109:637771): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.109:637771): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=9987007 dev=00:b3 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.109:637771): item=0 name=\"/usr/bin/cat\" inode=9986333 dev=00:b3 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.109:637771): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.109:637771): argc=2 a0=\"cat\" a1=\"/sys/fs/cgroup//memory.stat\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.109:637771): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.109:637771): arch=c000003e syscall=59 success=yes exit=0 a0=713178174d60 a1=7131186001e0 a2=60d6da619650 a3=8 items=2 ppid=5455 pid=1823440 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"cat\" exe=\"/usr/bin/cat\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.109:637770): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.109:637770): item=0 name=\"/usr/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.109:637770): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.109:637770): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=7131186001e0 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823440 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.109:637769): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.109:637769): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.109:637769): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.109:637769): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=7131186001e0 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823440 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.108:637768): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.108:637768): item=0 name=\"/usr/local/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.108:637768): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.108:637768): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=7131186001e0 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823440 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.108:637767): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.108:637767): item=0 name=\"/tmp/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.108:637767): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.108:637767): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=7131186001e0 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823440 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.108:637766): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.108:637766): item=0 name=\"/root/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.108:637766): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.108:637766): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=7131186001e0 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823440 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.108:637765): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.108:637765): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.108:637765): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.108:637765): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=7131186001e0 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823440 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.105:637764): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.105:637764): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=9987007 dev=00:b3 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.105:637764): item=0 name=\"/usr/bin/cat\" inode=9986333 dev=00:b3 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.105:637764): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.105:637764): argc=2 a0=\"cat\" a1=\"/sys/fs/cgroup//memory.current\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.105:637764): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.105:637764): arch=c000003e syscall=59 success=yes exit=0 a0=713178174d60 a1=713118600180 a2=60d6da619650 a3=8 items=2 ppid=5455 pid=1823439 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"cat\" exe=\"/usr/bin/cat\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.105:637763): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.105:637763): item=0 name=\"/usr/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.105:637763): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.105:637763): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713118600180 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823439 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.105:637762): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.105:637762): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.105:637762): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.105:637762): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713118600180 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823439 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.105:637761): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.105:637761): item=0 name=\"/usr/local/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.105:637761): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.105:637761): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713118600180 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823439 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.105:637760): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.105:637760): item=0 name=\"/tmp/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.105:637760): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.105:637760): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713118600180 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823439 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.105:637759): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.105:637759): item=0 name=\"/root/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.105:637759): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.105:637759): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713118600180 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823439 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.105:637758): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.105:637758): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.105:637758): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.105:637758): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713118600180 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823439 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.101:637757): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.101:637757): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=9987007 dev=00:b3 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.101:637757): item=0 name=\"/usr/bin/cat\" inode=9986333 dev=00:b3 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.101:637757): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.101:637757): argc=2 a0=\"cat\" a1=\"/proc/self/cgroup\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=BPRM_FCAPS msg=audit(1781049597.101:637757): fver=0 fp=0 fi=0 fe=0 old_pp=00000000a80425fb old_pi=0 old_pe=00000000a80425fb old_pa=0 pp=00000000a80425fb pi=0 pe=00000000a80425fb pa=0 frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.101:637757): arch=c000003e syscall=59 success=yes exit=0 a0=713178174d60 a1=713171000440 a2=60d6da619650 a3=8 items=2 ppid=5455 pid=1823433 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"cat\" exe=\"/usr/bin/cat\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.101:637756): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.101:637756): item=0 name=\"/usr/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.101:637756): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.101:637756): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713171000440 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823433 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.101:637755): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.101:637755): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.101:637755): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.101:637755): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713171000440 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823433 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.100:637754): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.100:637754): item=0 name=\"/usr/local/sbin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.100:637754): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.100:637754): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713171000440 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823433 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.100:637753): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.100:637753): item=0 name=\"/tmp/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.100:637753): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.100:637753): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713171000440 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823433 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.100:637752): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.100:637752): item=0 name=\"/root/.local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.100:637752): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.100:637752): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713171000440 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823433 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.100:637751): proctitle=\"windmill\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.100:637751): item=0 name=\"/usr/local/bin/cat\" nametype=UNKNOWN cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.100:637751): cwd=\"/usr/src/app\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.100:637751): arch=c000003e syscall=59 success=no exit=-2 a0=713178174d60 a1=713171000440 a2=60d6da619650 a3=8 items=1 ppid=5455 pid=1823433 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"tokio-runtime-w\" exe=\"/usr/src/app/windmill\" subj=docker-default key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049597.096:637750): proctitle=72756E63002D2D726F6F74002F7661722F72756E2F646F636B65722F72756E74696D652D72756E632F6D6F6279002D2D6C6F67002F72756E2F636F6E7461696E6572642F696F2E636F6E7461696E6572642E72756E74696D652E76322E7461736B2F6D6F62792F37656465303139653363623065383839303138636162386266"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.096:637750): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=4108210 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049597.096:637750): item=0 name=\"/usr/bin/runc\" inode=4092559 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049597.096:637750): cwd=\"/run/containerd/io.containerd.runtime.v2.task/moby/7ede019e3cb0e889018cab8bf0c09e5e0f9742f2ec242b172ed023e182cb56d7\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049597.096:637750): argc=15 a0=\"runc\" a1=\"--root\" a2=\"/var/run/docker/runtime-runc/moby\" a3=\"--log\" a4=\"/run/containerd/io.containerd.runtime.v2.task/moby/7ede019e3cb0e889018cab8bf0c09e5e0f9742f2ec242b172ed023e182cb56d7/log.json\" a5=\"--log-format\" a6=\"json\" a7=\"--systemd-cgroup\" a8=\"exec\" a9=\"--process\" a10=\"/tmp/runc-process3992845204\" a11=\"--detach\" a12=\"--pid-file\" a13=\"/run/containerd/io.containerd.runtime.v2.task/moby/7ede019e3cb0e889018cab8bf0c09e5e0f9742f2ec242b172ed023e182cb56d7/19f3398d3ead0f1a103aa4e6aee7a0a80c4a0f0bbd18dbcc1c14edf2dffc5a7f.pid\" a14=\"7ede019e3cb0e889018cab8bf0c09e5e0f9742f2ec242b172ed023e182cb56d7\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049597.096:637750): arch=c000003e syscall=59 success=yes exit=0 a0=c0000c8210 a1=c0000d9080 a2=c0000d9100 a3=0 items=2 ppid=4862 pid=1823432 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"runc\" exe=\"/usr/bin/runc\" subj=unconfined key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049596.990:637749): proctitle=72756E6300696E6974"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049596.990:637749): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=4108210 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049596.990:637749): item=0 name=\"/proc/self/fd/6\" inode=4092559 dev=00:bb mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049596.990:637749): cwd=\"/var/lib/docker/rootfs/overlayfs/d8657209894f56f3640212d72fbd7b3cd015708249d96b584cbbea09abc6e165\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049596.990:637749): argc=2 a0=\"runc\" a1=\"init\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049596.990:637749): arch=c000003e syscall=59 success=yes exit=0 a0=c0001f3820 a1=c0001f7338 a2=c0001f9c40 a3=0 items=2 ppid=1823413 pid=1823422 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"6\" exe=\"/runc\" subj=unconfined key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PROCTITLE msg=audit(1781049596.976:637748): proctitle=72756E63002D2D726F6F74002F7661722F72756E2F646F636B65722F72756E74696D652D72756E632F6D6F6279002D2D6C6F67002F72756E2F636F6E7461696E6572642F696F2E636F6E7461696E6572642E72756E74696D652E76322E7461736B2F6D6F62792F64383635373230393839346635366633363430323132643732"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049596.976:637748): item=1 name=\"/lib64/ld-linux-x86-64.so.2\" inode=4108210 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=PATH msg=audit(1781049596.976:637748): item=0 name=\"/usr/bin/runc\" inode=4092559 dev=fc:00 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0\u001dOUID=\"root\" OGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=CWD msg=audit(1781049596.976:637748): cwd=\"/run/containerd/io.containerd.runtime.v2.task/moby/d8657209894f56f3640212d72fbd7b3cd015708249d96b584cbbea09abc6e165\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=EXECVE msg=audit(1781049596.976:637748): argc=15 a0=\"runc\" a1=\"--root\" a2=\"/var/run/docker/runtime-runc/moby\" a3=\"--log\" a4=\"/run/containerd/io.containerd.runtime.v2.task/moby/d8657209894f56f3640212d72fbd7b3cd015708249d96b584cbbea09abc6e165/log.json\" a5=\"--log-format\" a6=\"json\" a7=\"--systemd-cgroup\" a8=\"exec\" a9=\"--process\" a10=\"/tmp/runc-process3134823008\" a11=\"--detach\" a12=\"--pid-file\" a13=\"/run/containerd/io.containerd.runtime.v2.task/moby/d8657209894f56f3640212d72fbd7b3cd015708249d96b584cbbea09abc6e165/74c82cb4666f4e63324cb71c8b8a846f82cda09997e0c7f89fbcbf126269069d.pid\" a14=\"d8657209894f56f3640212d72fbd7b3cd015708249d96b584cbbea09abc6e165\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "detected_level": "unknown", "filename": "/var/log/audit/audit.log", "job": "auditd", "service_name": "auditd", "source": "host"}, "msg": "type=SYSCALL msg=audit(1781049596.976:637748): arch=c000003e syscall=59 success=yes exit=0 a0=c0000107d0 a1=c000098c80 a2=c000098d00 a3=0 items=2 ppid=3897 pid=1823413 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"runc\" exe=\"/usr/bin/runc\" subj=unconfined key=\"root_commands\"\u001dARCH=x86_64 SYSCALL=execve AUID=\"unset\" UID=\"root\" GID=\"root\" EUID=\"root\" SUID=\"root\" FSUID=\"root\" EGID=\"root\" SGID=\"root\" FSGID=\"root\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-os-backend", "detected_level": "info", "service_name": "/prometheus-os-backend"}, "msg": "{\"time\":\"2026-06-09T23:59:57.229311344Z\",\"level\":\"INFO\",\"msg\":\"http request\",\"method\":\"GET\",\"path\":\"/health\",\"status\":200,\"duration_ms\":0,\"remote_addr\":\"[::1]:53538\"}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-windmill-server-1", "detected_level": "info", "service_name": "/prometheus-holding-windmill-server-1"}, "msg": "\u001b[2m2026-06-09T23:59:57.832975Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2mwindmill-worker/src/worker_utils.rs\u001b[0m\u001b[2m:\u001b[0m\u001b[2m84\u001b[0m\u001b[2m:\u001b[0m ping update, memory: container=176MB, windmill=176MB \u001b[3mworker\u001b[0m\u001b[2m=\u001b[0mwk-default-85b545c504cd-LTh8O \u001b[3mhostname\u001b[0m\u001b[2m=\u001b[0m85b545c504cd"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-windmill-server-1", "detected_level": "info", "service_name": "/prometheus-holding-windmill-server-1"}, "msg": "\u001b[2m2026-06-09T23:59:57.269097Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2msrc/main.rs\u001b[0m\u001b[2m:\u001b[0m\u001b[2m1541\u001b[0m\u001b[2m:\u001b[0m monitor task finished"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-windmill-server-1", "detected_level": "info", "service_name": "/prometheus-holding-windmill-server-1"}, "msg": "\u001b[2m2026-06-09T23:59:57.100637Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2msrc/main.rs\u001b[0m\u001b[2m:\u001b[0m\u001b[2m1522\u001b[0m\u001b[2m:\u001b[0m monitor task started"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-growthbook-postgres-1", "detected_level": "unknown", "service_name": "/prometheus-holding-growthbook-postgres-1"}, "msg": "2026-06-09 23:59:57.813 UTC [32] LOG:  cron job 3 COMMAND completed: CALL "}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-growthbook-postgres-1", "detected_level": "unknown", "service_name": "/prometheus-holding-growthbook-postgres-1"}, "msg": "2026-06-09 23:59:57.800 UTC [32] LOG:  cron job 3 starting: CALL documentdb_api_internal.build_index_concurrently(2);"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-growthbook-postgres-1", "detected_level": "unknown", "service_name": "/prometheus-holding-growthbook-postgres-1"}, "msg": "2026-06-09 23:59:57.459 UTC [32] LOG:  cron job 2 COMMAND completed: CALL "}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-growthbook-postgres-1", "detected_level": "unknown", "service_name": "/prometheus-holding-growthbook-postgres-1"}, "msg": "2026-06-09 23:59:57.443 UTC [32] LOG:  cron job 2 starting: CALL documentdb_api_internal.build_index_concurrently(1);"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-forgejo-1", "detected_level": "unknown", "service_name": "/prometheus-holding-forgejo-1"}, "msg": "2026/06/09 23:59:57 ...eb/routing/logger.go:102:func1() [I] router: completed POST /api/actions/runner.v1.RunnerService/FetchTask for 172.18.0.14:56338, 200 OK in 27.7ms @ <autogenerated>:1(http.Handler.ServeHTTP-fm)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/prometheus-holding-chromadb-1", "detected_level": "info", "service_name": "/prometheus-holding-chromadb-1"}, "msg": "INFO:     [09-06-2026 23:59:57] 127.0.0.1:40284 - \"GET /api/v1/heartbeat HTTP/1.1\" 200"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-windmill-worker", "detected_level": "info", "service_name": "/apex-windmill-worker"}, "msg": "\u001b[2m2026-06-09T23:59:57.150640Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2mwindmill-worker/src/worker_utils.rs\u001b[0m\u001b[2m:\u001b[0m\u001b[2m90\u001b[0m\u001b[2m:\u001b[0m ping update, memory: container=42MB, windmill=25MB, db_latency=36ms \u001b[3mworker\u001b[0m\u001b[2m=\u001b[0mwk-default-6852850569bd-3Oh12 \u001b[3mhostname\u001b[0m\u001b[2m=\u001b[0m6852850569bd"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-qdrant", "detected_level": "info", "service_name": "/apex-qdrant"}, "msg": "2026-06-09T23:59:58.676475Z  INFO actix_web::middleware::logger: 172.21.0.5 \"POST /collections/apex_knowledge/points/query HTTP/1.1\" 200 857 \"-\" \"python-client/1.18.0 python/3.11.15\" 0.023534    "}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-qdrant", "detected_level": "info", "service_name": "/apex-qdrant"}, "msg": "2026-06-09T23:59:58.599773Z  INFO actix_web::middleware::logger: 172.21.0.5 \"PUT /collections/apex_knowledge/points?wait=true HTTP/1.1\" 200 85 \"-\" \"python-client/1.18.0 python/3.11.15\" 0.022318    "}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:56 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "warn", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:56 - LiteLLM:WARNING\u001b[0m: utils.py:764 - `litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openai.common_utils.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 989, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise OpenAIError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/openai/_client.py\", line 496, in __init__"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        organization=organization,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        api_key=api_key,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                              ~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 385, in _get_openai_client"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        shared_session=shared_session,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<7 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        is_async=True,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                                  ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    openai_aclient: AsyncOpenAI = self._get_openai_client(  # type: ignore"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/openai/openai.py\", line 903, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<6 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise APIError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2389, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_fallbacks(**kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1699, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 1723, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "                ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    responses = await llm_responses"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/common_request_processing.py\", line 992, in base_process_llm_request"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<16 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await base_llm_response_processor.base_process_llm_request("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Final returned optional params: {'temperature': 0.0, 'stream': False, 'max_tokens': 4000, 'max_retries': 0, 'extra_body': {}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/proxy/proxy_server.py\", line 6817, in chat_completion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Initialized litellm callbacks, Async Success Callbacks: [<bound method Router.deployment_callback_on_success of <litellm.router.Router object at 0x77d3efaee900>>, <litellm.proxy.hooks.litellm_skills.main.SkillsInjectionHook object at 0x77d3fdc15940>, <litellm.proxy.hooks.model_max_budget_limiter._PROXY_VirtualKeyModelMaxBudgetLimiter object at 0x77d3f13af770>, <litellm.proxy.hooks.max_budget_limiter._PROXY_MaxBudgetLimiter object at 0x77d3efb19590>, <litellm.proxy.hooks.parallel_request_limiter_v3._PROXY_MaxParallelRequestsHandler_v3 object at 0x77d3efaef230>, <litellm.proxy.hooks.cache_control_check._PROXY_CacheControlCheck object at 0x77d3efb196d0>, <litellm.proxy.hooks.responses_id_security.ResponsesIDSecurity object at 0x77d3efaef380>, <litellm.proxy.hooks.max_iterations_limiter._PROXY_MaxIterationsHandler object at 0x77d3efaef4d0>, <litellm.proxy.hooks.max_budget_per_session_limiter._PROXY_MaxBudgetPerSessionHandler object at 0x77d3efaef620>, <litellm_enterprise.proxy.hooks.managed_files._PROXY_LiteLLMManagedFiles object at 0x77d3efaef770>, <litellm_enterprise.proxy.hooks.managed_vector_stores._PROXY_LiteLLMManagedVectorStores object at 0x77d3efaef8c0>, <litellm._service_logger.ServiceLogging object at 0x77d3f163a3f0>]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 200, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self._make_common_async_call("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 307, in async_completion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await init_response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 620, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise HTTPStatusError(message, request=request, response=self)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/httpx/_models.py\", line 829, in raise_for_status"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response.raise_for_status()"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 469, in post"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/http_handler.py\", line 513, in post"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await func(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/logging_utils.py\", line 297, in async_wrapper"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<10 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await async_httpx_client.post("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 175, in _make_common_async_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=['apex-fallback']"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Cooldown Deployments=[]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Debug Information:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Error doing the fallback: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Available Model Group Fallbacks=None"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.AuthenticationError: litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<5 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise AuthenticationError("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 510, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py\", line 2456, in exception_type"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        extra_kwargs=kwargs,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<3 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        model=model,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "          ~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise exception_type("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/main.py\", line 639, in acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    result = await original_function(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 1893, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/utils.py\", line 2094, in wrapper_async"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await _response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2166, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise e"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 2220, in _acompletion"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await response"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5754, in make_call"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.make_call(original_function, *args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5586, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        content_policy_fallbacks=content_policy_fallbacks,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<4 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5635, in async_function_with_retries"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await self.async_function_with_retries(*args, **kwargs)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5480, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5404, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    raise original_exception"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5446, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<8 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    return await self.async_function_with_fallbacks_common_utils("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5489, in async_function_with_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        ^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "        *args, **kwargs"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await litellm_router.async_function_with_fallbacks("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 139, in run_async_fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router_utils/fallback_event_handlers.py\", line 161, in run_async_fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    )"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    ...<2 lines>..."}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "               ^^^^^^^^^^^^^^^^^^^^^^^^^"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "    response = await run_async_fallback("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5413, in async_function_with_fallbacks_common_utils"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "Traceback (most recent call last):"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": "During handling of the above exception, another exception occurred:"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "unknown", "service_name": "/apex-litellm"}, "msg": ""}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34996 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34990 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34982 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34980 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34964 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34958 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34952 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34944 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34936 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "INFO:     172.21.0.5:34932 - \"POST /v1/chat/completions HTTP/1.1\" 402 Payment Required"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "info", "service_name": "/apex-litellm"}, "msg": "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:58 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:57 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:56 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:56 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]. Received Model Group=apex-fallback"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:56 - LiteLLM Router:ERROR\u001b[0m: router.py:5421 - litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - litellm.AuthenticationError: AuthenticationError: OpenAIException - The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variableNo fallback model group found for original model_group=apex-fallback. Fallbacks=[{'apex-tier1': ['apex-fallback']}, {'apex-tier2': ['apex-fallback']}, {'apex-tier3': ['apex-fallback']}]"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5839, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.llms.openrouter.common_utils.OpenRouterException: {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise provider_config.get_error_class("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/llms/custom_httpx/llm_http_handler.py\", line 4720, in _handle_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise self._handle_error(e=e, provider_config=provider_config)"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "httpx.HTTPStatusError: Client error '402 Payment Required' for url 'https://openrouter.ai/api/v1/chat/completions'"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "\u001b[92m23:59:56 - LiteLLM Proxy:ERROR\u001b[0m: common_request_processing.py:1495 - litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}. Received Model Group=apex-tier2"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error  # then raise error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "  File \"/usr/lib/python3.13/site-packages/litellm/router.py\", line 5860, in should_retry_this_error"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "        error=e,"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    self.should_retry_this_error("}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "    raise error_from_fallbacks"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-litellm", "detected_level": "error", "service_name": "/apex-litellm"}, "msg": "litellm.exceptions.APIError: litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Insufficient credits. Add more using https://openrouter.ai/settings/credits\",\"code\":402}}"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "_parse_tags: raw input is empty, returning defaults"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed in pipeline, applying fallback mode."}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:58Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:57Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}
{"ts": "2026-06-09T23:59:56Z", "labels": {"cluster": "docker-compose", "container": "/apex-backend", "detected_level": "unknown", "service_name": "/apex-backend"}, "msg": "LLM call failed: 402 Client Error: Payment Required for url: http://apex-litellm:4000/v1/chat/completions"}