Geo proxied git fetch error: fatal: the remote end hung up unexpectedly (#454707) · Issues · GitLab.org / GitLab · GitLab (original) (raw)
Geo proxied git fetch error: fatal: the remote end hung up unexpectedly
Problem
From https://gitlab.com/gitlab-com/geo-customers/-/issues/206
Running of
git fetch(with no extra options) over ssh to secondary hangs and times out after 60 seconds when request is proxied to primary. This is reproducible.
git cloneover ssh however succeeds when proxied. Also reproducible.The fetch error doesn't occur when the repo is small (e.g. just a README).
On git client we see:
➜ gitlab-runner git:(main) git pull
remote:
remote: This request to a Geo secondary node will be forwarded to the
remote: Geo primary node:
remote:
remote: git@staging-ref.gitlab.com:mkozonotest/gitlab-runner.git
remote:
remote:
remote: ========================================================================
remote:
remote: ERROR: Remote repository is unavailable
remote:
remote: ========================================================================
remote:
fatal: the remote end hung up unexpectedlyClick here to expand staging-ref logs with correlation ID 01HTQV6DCA5NEH56B7QKKF00PM
[
{
"insertId": "ewysgjyqsr3ydl8v",
"jsonPayload": {
"msg": "server: handleConn: start",
"level": "info",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM"
},
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"location": "europe-west6-c",
"container_name": "gitlab-shell",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-eu"
}
},
"timestamp": "2024-04-05T19:19:01Z",
"severity": "INFO",
"labels": {
"k8s-pod/heritage": "Helm",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/release": "gitlab"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:02.293495009Z"
},
{
"insertId": "3x0k0rixp09dqjem",
"jsonPayload": {
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"ssh_key_type": "ssh-ed25519",
"msg": "public key authentication",
"level": "info"
},
"resource": {
"type": "k8s_container",
"labels": {
"cluster_name": "staging-ref-3k-hybrid-eu",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"location": "europe-west6-c",
"container_name": "gitlab-shell",
"project_id": "gitlab-staging-ref",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:02Z",
"severity": "INFO",
"labels": {
"k8s-pod/app": "gitlab-shell",
"k8s-pod/heritage": "Helm",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/release": "gitlab",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "4dvkxpz7mwm361dd",
"jsonPayload": {
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"duration_ms": 30,
"msg": "Internal API error",
"status": 404,
"url": "http://gitlab-webservice-default.default.svc:8181/api/v4/internal/authorized_keys?key=AAAAC3NzaC1lZDI1NTE5AAAAIPDWtv2nCaZPjcm6iSgxSnVPVvQbJjAa82UuRbe9UvJB",
"method": "GET",
"error": null,
"level": "error"
},
"resource": {
"type": "k8s_container",
"labels": {
"cluster_name": "staging-ref-3k-hybrid-eu",
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"namespace_name": "default",
"container_name": "gitlab-shell",
"location": "europe-west6-c"
}
},
"timestamp": "2024-04-05T19:19:02Z",
"severity": "ERROR",
"labels": {
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/release": "gitlab",
"k8s-pod/heritage": "Helm"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "1p0e7x055cfobgbl",
"jsonPayload": {
"ssh_key_type": "ssh-ed25519",
"msg": "public key authentication",
"level": "info",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM"
},
"resource": {
"type": "k8s_container",
"labels": {
"location": "europe-west6-c",
"project_id": "gitlab-staging-ref",
"container_name": "gitlab-shell",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"cluster_name": "staging-ref-3k-hybrid-eu",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:03Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/heritage": "Helm",
"k8s-pod/release": "gitlab",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/pod-template-hash": "69fc6cd8c4"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "7hayz77093snsvqa",
"jsonPayload": {
"msg": "public key authentication",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"level": "info",
"ssh_key_type": "ssh-ed25519"
},
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "gitlab-shell",
"location": "europe-west6-c",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"cluster_name": "staging-ref-3k-hybrid-eu",
"namespace_name": "default",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:03Z",
"severity": "INFO",
"labels": {
"k8s-pod/app": "gitlab-shell",
"k8s-pod/release": "gitlab",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/heritage": "Helm"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "dogm2v95hoxd0jyd",
"jsonPayload": {
"msg": "public key authentication",
"level": "info",
"ssh_key_type": "ssh-ed25519",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM"
},
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-eu",
"location": "europe-west6-c",
"container_name": "gitlab-shell",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb"
}
},
"timestamp": "2024-04-05T19:19:03Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/heritage": "Helm",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/pod-template-hash": "69fc6cd8c4"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "f33e5lve9ilxk056",
"jsonPayload": {
"method": "GET",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"url": "http://gitlab-webservice-default.default.svc:8181/api/v4/internal/authorized_keys?key=AAAAC3NzaC1lZDI1NTE5AAAAIBvfbhosc9J7qpkWelsT1ABynJRxQ%2F82dQjMa0jJEY38",
"status": 404,
"error": null,
"duration_ms": 55,
"level": "error",
"msg": "Internal API error"
},
"resource": {
"type": "k8s_container",
"labels": {
"location": "europe-west6-c",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"container_name": "gitlab-shell",
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-eu",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:03Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/heritage": "Helm",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/release": "gitlab"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "t9jue50hzwle8ki9",
"jsonPayload": {
"status": 200,
"method": "GET",
"url": "http://gitlab-webservice-default.default.svc:8181/api/v4/internal/authorized_keys?key=AAAAC3NzaC1lZDI1NTE5AAAAIFGdPz2ptDG8I%2F5dEBzq6NCQ0DDyOCHajzuFmZDinlpi",
"content_length_bytes": 285,
"msg": "Finished HTTP request",
"duration_ms": 171,
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"level": "info"
},
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"location": "europe-west6-c",
"container_name": "gitlab-shell",
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-eu",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:03Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/heritage": "Helm",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "w1sfewemwjfdpgmt",
"jsonPayload": {
"method": "GET",
"msg": "Internal API error",
"duration_ms": 26,
"status": 404,
"error": null,
"url": "http://gitlab-webservice-default.default.svc:8181/api/v4/internal/authorized_keys?key=AAAAC3NzaC1lZDI1NTE5AAAAICQxYYIeb5%2FS24BiRpA6MywkiPYZLLIcc6vgOrAnit48",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"level": "error"
},
"resource": {
"type": "k8s_container",
"labels": {
"cluster_name": "staging-ref-3k-hybrid-eu",
"project_id": "gitlab-staging-ref",
"location": "europe-west6-c",
"namespace_name": "default",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"container_name": "gitlab-shell"
}
},
"timestamp": "2024-04-05T19:19:03Z",
"severity": "ERROR",
"labels": {
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/release": "gitlab",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/heritage": "Helm"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "c8cgdzqmu3hhg5s3",
"jsonPayload": {
"level": "info",
"env": {
"GitProtocolVersion": "version=2",
"RemoteAddr": "10.92.6.193:47566",
"OriginalCommand": "git-upload-pack 'mkozonotest/gitlab-runner.git'",
"IsSSHConnection": true,
"NamespacePath": ""
},
"command": "*uploadpack.Command",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"established_session_duration_s": 0.214360329,
"msg": "session: handleShell: executing command"
},
"resource": {
"type": "k8s_container",
"labels": {
"location": "europe-west6-c",
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"cluster_name": "staging-ref-3k-hybrid-eu",
"namespace_name": "default",
"container_name": "gitlab-shell"
}
},
"timestamp": "2024-04-05T19:19:04Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/heritage": "Helm",
"k8s-pod/app": "gitlab-shell",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "sf95qna2o0t8ls7z",
"jsonPayload": {
"channel_type": "session",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"remote_addr": "10.92.6.193:47566",
"msg": "connection: handle: new channel requested",
"level": "info"
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"container_name": "gitlab-shell",
"cluster_name": "staging-ref-3k-hybrid-eu",
"project_id": "gitlab-staging-ref",
"location": "europe-west6-c"
}
},
"timestamp": "2024-04-05T19:19:04Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/app": "gitlab-shell",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/heritage": "Helm"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "9zmsty5r1avt7g72",
"jsonPayload": {
"duration_ms": 1206,
"url": "http://gitlab-webservice-default.default.svc:8181/api/v4/internal/allowed",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"content_length_bytes": 933,
"method": "POST",
"status": 300,
"msg": "Finished HTTP request",
"level": "info"
},
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "gitlab-shell",
"namespace_name": "default",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"cluster_name": "staging-ref-3k-hybrid-eu",
"project_id": "gitlab-staging-ref",
"location": "europe-west6-c"
}
},
"timestamp": "2024-04-05T19:19:05Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/heritage": "Helm",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/app": "gitlab-shell"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "s3nhyz0tly03pmdu",
"jsonPayload": {
"method": "GET",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"level": "info",
"status": 200,
"msg": "Finished HTTP request",
"duration_ms": 756,
"url": "https://staging-ref.gitlab.com/mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack"
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-eu",
"container_name": "gitlab-shell",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"location": "europe-west6-c",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:06Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/heritage": "Helm"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:07.244016841Z"
},
{
"insertId": "030xzsubiohy5cf4",
"jsonPayload": {
"remote_addr": "10.92.6.193:47566",
"duration_s": 63.404141218,
"msg": "connection: handleRequests: done",
"level": "info",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM"
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-eu",
"container_name": "gitlab-shell",
"location": "europe-west6-c",
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb"
}
},
"timestamp": "2024-04-05T19:20:07Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/app": "gitlab-shell",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/heritage": "Helm"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:20:12.271680196Z"
},
{
"insertId": "drlaxflota7yb60k",
"jsonPayload": {
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"exit_status": 1,
"level": "info",
"msg": "session: exit: exiting"
},
"resource": {
"type": "k8s_container",
"labels": {
"cluster_name": "staging-ref-3k-hybrid-eu",
"location": "europe-west6-c",
"container_name": "gitlab-shell",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"namespace_name": "default",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:20:07Z",
"severity": "INFO",
"labels": {
"k8s-pod/chart": "gitlab-shell-7.10.1",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/app": "gitlab-shell",
"k8s-pod/heritage": "Helm",
"k8s-pod/release": "gitlab"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:20:12.271680196Z"
},
{
"insertId": "lqxrjj4o5vqzh15i",
"jsonPayload": {
"url": "https://staging-ref.gitlab.com/mkozonotest/gitlab-runner.git/git-upload-pack",
"msg": "Internal API unreachable",
"method": "POST",
"level": "error",
"error": "http2: Transport: cannot retry err [stream error: stream ID 1; PROTOCOL_ERROR; received from peer] after Request.Body was written; define Request.GetBody to avoid this error",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"duration_ms": 61017
},
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"location": "europe-west6-c",
"namespace_name": "default",
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-eu",
"container_name": "gitlab-shell"
}
},
"timestamp": "2024-04-05T19:20:07Z",
"severity": "ERROR",
"labels": {
"k8s-pod/chart": "gitlab-shell-7.10.1",
"k8s-pod/release": "gitlab",
"k8s-pod/heritage": "Helm",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/app": "gitlab-shell"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:20:12.271680196Z"
},
{
"insertId": "8tjcjud7szuucfe2",
"jsonPayload": {
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"level": "info",
"meta": {},
"written_bytes": 0,
"duration_s": 66.746373664,
"msg": "access: finish",
"remote_addr": "10.92.6.193:47566"
},
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"project_id": "gitlab-staging-ref",
"namespace_name": "default",
"location": "europe-west6-c",
"cluster_name": "staging-ref-3k-hybrid-eu",
"container_name": "gitlab-shell"
}
},
"timestamp": "2024-04-05T19:20:08Z",
"severity": "INFO",
"labels": {
"k8s-pod/heritage": "Helm",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/release": "gitlab",
"k8s-pod/app": "gitlab-shell"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:20:12.271680196Z"
},
{
"insertId": "bxqj9pe711c6djac",
"jsonPayload": {
"level": "info",
"reason": "ssh: disconnect, reason 11: disconnected by user",
"correlation_id": "01HTQV6DCA5NEH56B7QKKF00PM",
"msg": "server: handleConn: done"
},
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-eu",
"namespace_name": "default",
"location": "europe-west6-c",
"pod_name": "gitlab-gitlab-shell-69fc6cd8c4-dw9zb",
"container_name": "gitlab-shell"
}
},
"timestamp": "2024-04-05T19:20:08Z",
"severity": "INFO",
"labels": {
"k8s-pod/app": "gitlab-shell",
"k8s-pod/pod-template-hash": "69fc6cd8c4",
"k8s-pod/heritage": "Helm",
"k8s-pod/chart": "gitlab-shell-7.10.1",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-2a286a14-ph98",
"k8s-pod/release": "gitlab"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:20:12.271680196Z"
}
]Click here to expand one additional log on the secondary site for POST /api/:version/internal/allowed
{
"insertId": "xtxck51892e9foou",
"jsonPayload": {
"meta.feature_category": "source_code_management",
"db_primary_wal_count": 0,
"db_main_replica_cached_count": 3,
"redis_feature_flag_write_bytes": 603,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_cached_count": 0,
"redis_rate_limiting_read_bytes": 679,
"meta.project": "mkozonotest/gitlab-runner",
"db_geo_wal_count": 0,
"view_duration_s": 0.81527,
"meta.caller_id": "POST /api/:version/internal/allowed",
"duration_s": 1.16612,
"db_ci_wal_cached_count": 0,
"db_primary_cached_count": 0,
"db_replica_duration_s": 0.186,
"ua": "GitLab-Shell",
"db_write_count": 0,
"db_txn_count": 0,
"redis_feature_flag_calls": 6,
"db_duration_s": 0.35085,
"redis_write_bytes": 1349,
"db_primary_txn_max_duration_s": 0,
"redis_read_bytes": 2149,
"db_replica_txn_duration_s": 0,
"content_length": "145",
"db_main_wal_cached_count": 0,
"db_geo_cached_count": 0,
"db_ci_count": 0,
"redis_db_load_balancing_read_bytes": 675,
"mem_mallocs": 301414,
"remote_ip": "10.92.6.186",
"db_main_replica_wal_cached_count": 0,
"mem_objects": 536160,
"target_duration_s": 1,
"worker_id": "puma_3",
"request_urgency": "default",
"db_main_txn_count": 0,
"redis_feature_flag_duration_s": 0.003393,
"params": [
{
"value": "git-upload-pack",
"key": "action"
},
{
"value": "mkozonotest/gitlab-runner.git",
"key": "project"
},
{
"key": "changes",
"value": "_any"
},
{
"value": "ssh",
"key": "protocol"
},
{
"value": "3410",
"key": "key_id"
},
{
"value": "10.92.6.193",
"key": "check_ip"
}
],
"redis_rate_limiting_calls": 8,
"db_main_duration_s": 0.002,
"db_replica_wal_cached_count": 0,
"db_ci_replica_txn_max_duration_s": 0,
"db_primary_count": 2,
"redis_db_load_balancing_duration_s": 0.002532,
"db_main_cached_count": 0,
"meta.remote_ip": "10.92.6.186",
"redis_db_load_balancing_calls": 5,
"db_ci_txn_duration_s": 0,
"db_replica_txn_max_duration_s": 0,
"db_geo_txn_count": 0,
"db_ci_replica_wal_count": 0,
"method": "POST",
"db_ci_duration_s": 0,
"mem_bytes": 55278968,
"db_ci_replica_duration_s": 0,
"meta.root_namespace": "mkozonotest",
"db_ci_txn_max_duration_s": 0,
"meta.client_id": "user/6602",
"redis_rate_limiting_duration_s": 0.003894,
"db_main_replica_duration_s": 0.186,
"db_replica_count": 15,
"subcomponent": "api_json",
"db_count": 17,
"redis_duration_s": 0.009819,
"redis_feature_flag_read_bytes": 795,
"db_primary_duration_s": 0.003,
"db_main_replica_txn_max_duration_s": 0,
"db_replica_wal_count": 0,
"db_main_wal_count": 0,
"db_geo_count": 1,
"pid": 2445,
"db_ci_wal_count": 0,
"route": "/api/:version/internal/allowed",
"db_cached_count": 3,
"cpu_s": 0.837226,
"path": "/api/v4/internal/allowed",
"db_main_txn_duration_s": 0,
"db_main_count": 1,
"meta.user_id": 6602,
"component": "gitlab",
"host": "gitlab-webservice-default.default.svc",
"db_replica_txn_count": 0,
"db_ci_replica_count": 0,
"queue_duration_s": 0.015254,
"db_ci_replica_txn_count": 0,
"redis_db_load_balancing_write_bytes": 164,
"db_ci_txn_count": 0,
"mem_total_bytes": 76725368,
"correlation_id": "01HTQV6GMWBEVR64K16QHK2H5A",
"db_main_replica_count": 15,
"db_ci_cached_count": 0,
"db_main_replica_txn_count": 0,
"db_ci_replica_cached_count": 0,
"db_primary_txn_duration_s": 0,
"db_main_replica_txn_duration_s": 0,
"status": 300,
"redis_calls": 19,
"redis_rate_limiting_write_bytes": 582,
"db_geo_duration_s": 0.001,
"rate_limiting_gates": [
"gitlab_shell_operation"
],
"db_replica_cached_count": 3,
"db_geo_txn_max_duration_s": 0,
"db_main_txn_max_duration_s": 0,
"meta.user": "mkozono",
"db_primary_wal_cached_count": 0,
"db_geo_wal_cached_count": 0,
"db_primary_txn_count": 0,
"db_geo_txn_duration_s": 0,
"db_ci_replica_txn_duration_s": 0
},
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-eu",
"namespace_name": "default",
"container_name": "webservice",
"pod_name": "gitlab-webservice-default-7846fc87db-6t9rq",
"location": "europe-west6-c"
}
},
"timestamp": "2024-04-05T19:19:05.800Z",
"severity": "INFO",
"labels": {
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/pod-template-hash": "7846fc87db",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-6ffab6c8-4mjm",
"k8s-pod/heritage": "Helm",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/app": "webservice",
"k8s-pod/release": "gitlab"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:06.238528328Z"
}Click here to expand logs on the primary site
Logs during the timeframe of the problem, matching this:
SEARCH("mkozonotest")
resource.labels.cluster_name="staging-ref-3k-hybrid-us"[
{
"insertId": "209q6fo56xv0s1cy",
"jsonPayload": {
"remote_addr": "34.65.122.187:36968",
"msg": "access",
"referrer": "",
"correlation_id": "01HTQV6DPNKA1GXFP36EPHJMSC",
"host": "staging-ref.gitlab.com",
"content_type": "application/x-git-upload-pack-result",
"uri": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"route": "^/.+\\.git/git-upload-pack\\z",
"level": "info",
"method": "POST",
"system": "http",
"remote_ip": "34.65.122.187",
"duration_ms": 76,
"status": 200,
"written_bytes": 113625,
"ttfb_ms": 75,
"user_agent": "git/2.43.2",
"proto": "HTTP/1.1"
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"container_name": "gitlab-workhorse",
"cluster_name": "staging-ref-3k-hybrid-us",
"location": "us-east1-c"
}
},
"timestamp": "2024-04-05T19:19:01Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/shard": "default",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/app": "webservice",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/tier": "sv",
"k8s-pod/heritage": "Helm",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/stage": "main",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/type": "web"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:05.890135794Z"
},
{
"insertId": "6ceptwgdp7kqxui0",
"jsonPayload": {
"method": "GET",
"duration_ms": 62,
"remote_ip": "34.65.234.178",
"remote_addr": "34.65.234.178:45134",
"level": "info",
"msg": "access",
"content_type": "application/x-git-upload-pack-advertisement",
"written_bytes": 160,
"host": "staging-ref.gitlab.com",
"ttfb_ms": 55,
"system": "http",
"route": "^/.+\\.git/info/refs\\z",
"referrer": "",
"status": 200,
"proto": "HTTP/1.1",
"uri": "/mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack",
"user_agent": "git/2.43.2",
"correlation_id": "01HTQV6DHSTQDCGA9AV26TDKRB"
},
"resource": {
"type": "k8s_container",
"labels": {
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"namespace_name": "default",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"project_id": "gitlab-staging-ref",
"container_name": "gitlab-workhorse"
}
},
"timestamp": "2024-04-05T19:19:01Z",
"severity": "INFO",
"labels": {
"k8s-pod/stage": "main",
"k8s-pod/shard": "default",
"k8s-pod/type": "web",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/tier": "sv",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/release": "gitlab",
"k8s-pod/app": "webservice",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/heritage": "Helm",
"k8s-pod/chart": "webservice-7.10.1"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:05.890135794Z"
},
{
"insertId": "kksvkbt0zqdy1lma",
"jsonPayload": {
"status": 200,
"system": "http",
"correlation_id": "01HTQV6DPNQ760JPFYCN96G41P",
"written_bytes": 113625,
"user_agent": "git/2.43.2",
"referrer": "",
"duration_ms": 90,
"host": "staging-ref.gitlab.com",
"proto": "HTTP/1.1",
"msg": "access",
"ttfb_ms": 89,
"content_type": "application/x-git-upload-pack-result",
"level": "info",
"uri": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"route": "^/.+\\.git/git-upload-pack\\z",
"remote_addr": "34.65.5.0:58606",
"remote_ip": "34.65.5.0",
"method": "POST"
},
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "gitlab-workhorse",
"project_id": "gitlab-staging-ref",
"namespace_name": "default",
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"pod_name": "gitlab-webservice-default-669cf4599-qww8d"
}
},
"timestamp": "2024-04-05T19:19:01Z",
"severity": "INFO",
"labels": {
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/stage": "main",
"k8s-pod/app": "webservice",
"k8s-pod/tier": "sv",
"k8s-pod/release": "gitlab",
"k8s-pod/heritage": "Helm",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/type": "web",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/shard": "default"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.930552910Z"
},
{
"insertId": "pm4tisojlfets058",
"jsonPayload": {
"status": 200,
"level": "info",
"msg": "access",
"correlation_id": "01HTQV6DH9T3V2SEV6HCBV31H8",
"ttfb_ms": 61,
"host": "staging-ref.gitlab.com",
"user_agent": "git/2.43.2",
"referrer": "",
"content_type": "application/x-git-upload-pack-advertisement",
"proto": "HTTP/1.1",
"uri": "/mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack",
"method": "GET",
"remote_addr": "34.65.5.0:58606",
"duration_ms": 67,
"system": "http",
"written_bytes": 160,
"route": "^/.+\\.git/info/refs\\z",
"remote_ip": "34.65.5.0"
},
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "gitlab-workhorse",
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-us",
"location": "us-east1-c",
"pod_name": "gitlab-webservice-default-669cf4599-qww8d",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:01Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/app": "webservice",
"k8s-pod/heritage": "Helm",
"k8s-pod/type": "web",
"k8s-pod/shard": "default",
"k8s-pod/tier": "sv",
"k8s-pod/release": "gitlab",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/stage": "main",
"k8s-pod/pod-template-hash": "669cf4599"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.930552910Z"
},
{
"insertId": "r1yn3ri5mzdiskx2",
"jsonPayload": {
"level": "info",
"content_type": "application/x-git-upload-pack-result",
"remote_ip": "34.65.234.178",
"route": "^/.+\\.git/git-upload-pack\\z",
"system": "http",
"duration_ms": 68,
"referrer": "",
"uri": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"ttfb_ms": 66,
"remote_addr": "34.65.234.178:45134",
"proto": "HTTP/1.1",
"written_bytes": 113625,
"status": 200,
"msg": "access",
"user_agent": "git/2.43.2",
"correlation_id": "01HTQV6DQ0K7KSHFVR4KC8D5C2",
"method": "POST",
"host": "staging-ref.gitlab.com"
},
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "gitlab-workhorse",
"namespace_name": "default",
"location": "us-east1-c",
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"cluster_name": "staging-ref-3k-hybrid-us"
}
},
"timestamp": "2024-04-05T19:19:01Z",
"severity": "INFO",
"labels": {
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/app": "webservice",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/tier": "sv",
"k8s-pod/type": "web",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/shard": "default",
"k8s-pod/heritage": "Helm",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/release": "gitlab",
"k8s-pod/stage": "main"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:05.890135794Z"
},
{
"insertId": "wnitnyhtx0cegjab",
"jsonPayload": {
"method": "GET",
"duration_ms": 68,
"user_agent": "git/2.43.2",
"level": "info",
"remote_ip": "34.65.122.187",
"content_type": "application/x-git-upload-pack-advertisement",
"host": "staging-ref.gitlab.com",
"proto": "HTTP/1.1",
"uri": "/mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack",
"correlation_id": "01HTQV6DH80Y7B9JFMJMD1PAAG",
"written_bytes": 160,
"msg": "access",
"system": "http",
"route": "^/.+\\.git/info/refs\\z",
"referrer": "",
"ttfb_ms": 62,
"remote_addr": "34.65.122.187:36968",
"status": 200
},
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "gitlab-workhorse",
"namespace_name": "default",
"location": "us-east1-c",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx"
}
},
"timestamp": "2024-04-05T19:19:01Z",
"severity": "INFO",
"labels": {
"k8s-pod/type": "web",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/heritage": "Helm",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/release": "gitlab",
"k8s-pod/shard": "default",
"k8s-pod/gitlab_com/webservice-name": "default",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/stage": "main",
"k8s-pod/app": "webservice",
"k8s-pod/tier": "sv"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:05.890135794Z"
},
{
"insertId": "z6vyqpmgl5i8v2zd",
"jsonPayload": {
"target_duration_s": 1,
"db_main_replica_cached_count": 1,
"db_main_cached_count": 0,
"redis_calls": 2,
"db_primary_count": 0,
"meta.project": "mkozonotest/gitlab-runner",
"redis_write_bytes": 108,
"redis_repository_cache_duration_s": 0.00057,
"mem_total_bytes": 1334576,
"db_primary_txn_duration_s": 0,
"db_ci_cached_count": 0,
"db_main_duration_s": 0,
"db_ci_txn_max_duration_s": 0,
"db_replica_count": 10,
"meta.caller_id": "Repositories::GitHttpController#info_refs",
"db_ci_wal_count": 0,
"redis_feature_flag_read_bytes": 199,
"db_cached_count": 1,
"meta.client_id": "ip/34.65.122.187",
"meta.remote_ip": "34.65.122.187",
"pid": 469,
"db_main_replica_count": 10,
"db_main_replica_txn_count": 0,
"db_replica_txn_max_duration_s": 0,
"controller": "Repositories::GitHttpController",
"db_ci_duration_s": 0,
"db_txn_count": 0,
"duration_s": 0.04114,
"db_ci_txn_duration_s": 0,
"meta.root_namespace": "mkozonotest",
"params": [
{
"key": "service",
"value": "git-upload-pack"
},
{
"key": "repository_path",
"value": "mkozonotest/gitlab-runner.git"
}
],
"path": "/mkozonotest/gitlab-runner.git/info/refs",
"mem_objects": 12936,
"db_ci_replica_txn_count": 0,
"subcomponent": "production_json",
"db_ci_replica_wal_count": 0,
"mem_mallocs": 2935,
"db_main_txn_count": 0,
"db_main_replica_txn_duration_s": 0,
"redis_duration_s": 0.001142,
"db_ci_replica_txn_max_duration_s": 0,
"cpu_s": 0.037378,
"db_main_txn_max_duration_s": 0,
"remote_ip": "34.65.122.187",
"db_replica_duration_s": 0.011,
"redis_feature_flag_calls": 1,
"db_main_txn_duration_s": 0,
"db_replica_wal_cached_count": 0,
"db_main_wal_count": 0,
"redis_feature_flag_write_bytes": 53,
"db_main_replica_txn_max_duration_s": 0,
"db_duration_s": 0.01088,
"repository_storage": "default",
"db_primary_wal_count": 0,
"view_duration_s": 0.0003,
"db_ci_replica_count": 0,
"ua": "git/2.43.2",
"db_ci_replica_duration_s": 0,
"db_write_count": 0,
"db_main_replica_duration_s": 0.011,
"action": "info_refs",
"db_primary_wal_cached_count": 0,
"db_ci_count": 0,
"db_replica_cached_count": 1,
"db_primary_txn_max_duration_s": 0,
"redis_feature_flag_duration_s": 0.000572,
"db_main_wal_cached_count": 0,
"db_primary_txn_count": 0,
"format": "*/*",
"db_primary_duration_s": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"method": "GET",
"worker_id": "puma_2",
"db_count": 10,
"redis_repository_cache_calls": 1,
"component": "gitlab",
"db_ci_replica_wal_cached_count": 0,
"status": 200,
"db_ci_txn_count": 0,
"rate_limiting_gates": [],
"redis_repository_cache_write_bytes": 55,
"db_main_count": 0,
"db_replica_wal_count": 0,
"db_main_replica_wal_count": 0,
"correlation_id": "01HTQV6DH80Y7B9JFMJMD1PAAG",
"db_ci_replica_cached_count": 0,
"redis_read_bytes": 300,
"request_urgency": "default",
"db_replica_txn_count": 0,
"redis_repository_cache_read_bytes": 101,
"db_replica_txn_duration_s": 0,
"db_ci_replica_txn_duration_s": 0,
"meta.feature_category": "source_code_management",
"mem_bytes": 817136,
"db_primary_cached_count": 0
},
"resource": {
"type": "k8s_container",
"labels": {
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "webservice",
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:01.469Z",
"severity": "ERROR",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/stage": "main",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/type": "web",
"k8s-pod/app": "webservice",
"k8s-pod/tier": "sv",
"k8s-pod/heritage": "Helm",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/shard": "default",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:05.896021723Z"
},
{
"insertId": "njmo1cc4dmi6p13q",
"jsonPayload": {
"remote_ip": "34.65.5.0",
"db_primary_wal_count": 0,
"db_main_wal_cached_count": 0,
"method": "GET",
"ua": "git/2.43.2",
"db_replica_txn_duration_s": 0,
"db_primary_count": 0,
"db_replica_duration_s": 0.011,
"view_duration_s": 0.00025,
"db_ci_replica_count": 0,
"redis_repository_cache_duration_s": 0.000503,
"db_txn_count": 0,
"db_main_replica_wal_count": 0,
"db_primary_txn_duration_s": 0,
"db_ci_replica_wal_count": 0,
"db_ci_txn_duration_s": 0,
"db_primary_duration_s": 0,
"db_ci_replica_txn_count": 0,
"db_ci_count": 0,
"target_duration_s": 1,
"db_ci_replica_cached_count": 0,
"db_write_count": 0,
"cpu_s": 0.042221,
"db_ci_txn_count": 0,
"controller": "Repositories::GitHttpController",
"db_main_replica_txn_duration_s": 0,
"db_ci_replica_duration_s": 0,
"db_replica_wal_cached_count": 0,
"db_ci_wal_count": 0,
"db_ci_replica_txn_duration_s": 0,
"db_main_replica_duration_s": 0.011,
"db_primary_wal_cached_count": 0,
"format": "*/*",
"meta.caller_id": "Repositories::GitHttpController#info_refs",
"request_urgency": "default",
"db_main_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_count": 10,
"db_main_txn_count": 0,
"redis_read_bytes": 101,
"db_cached_count": 1,
"redis_calls": 1,
"path": "/mkozonotest/gitlab-runner.git/info/refs",
"db_main_wal_count": 0,
"mem_mallocs": 2870,
"db_ci_wal_cached_count": 0,
"db_replica_count": 10,
"worker_id": "puma_2",
"db_primary_txn_count": 0,
"db_ci_replica_txn_max_duration_s": 0,
"db_replica_wal_count": 0,
"redis_repository_cache_read_bytes": 101,
"mem_total_bytes": 1315912,
"db_ci_replica_wal_cached_count": 0,
"db_main_replica_txn_count": 0,
"redis_repository_cache_calls": 1,
"db_primary_cached_count": 0,
"meta.project": "mkozonotest/gitlab-runner",
"status": 200,
"mem_objects": 12685,
"db_main_duration_s": 0,
"correlation_id": "01HTQV6DH9T3V2SEV6HCBV31H8",
"db_main_txn_max_duration_s": 0,
"component": "gitlab",
"db_main_replica_count": 10,
"db_ci_cached_count": 0,
"db_replica_txn_max_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"params": [
{
"key": "service",
"value": "git-upload-pack"
},
{
"value": "mkozonotest/gitlab-runner.git",
"key": "repository_path"
}
],
"mem_bytes": 808512,
"db_main_cached_count": 0,
"meta.remote_ip": "34.65.5.0",
"redis_write_bytes": 55,
"db_main_replica_txn_max_duration_s": 0,
"action": "info_refs",
"redis_repository_cache_write_bytes": 55,
"db_ci_duration_s": 0,
"duration_s": 0.0423,
"meta.feature_category": "source_code_management",
"db_main_replica_cached_count": 1,
"redis_duration_s": 0.000503,
"pid": 529,
"rate_limiting_gates": [],
"repository_storage": "default",
"meta.root_namespace": "mkozonotest",
"db_replica_txn_count": 0,
"db_replica_cached_count": 1,
"meta.client_id": "ip/34.65.5.0",
"subcomponent": "production_json",
"db_duration_s": 0.01064,
"db_ci_txn_max_duration_s": 0,
"db_main_txn_duration_s": 0
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"pod_name": "gitlab-webservice-default-669cf4599-qww8d",
"container_name": "webservice",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:01.472Z",
"severity": "ERROR",
"labels": {
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/heritage": "Helm",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/stage": "main",
"k8s-pod/shard": "default",
"k8s-pod/type": "web",
"k8s-pod/release": "gitlab",
"k8s-pod/app": "webservice",
"k8s-pod/tier": "sv",
"k8s-pod/gitlab_com/webservice-name": "default"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:04.927265209Z"
},
{
"insertId": "ebrr527ynd4cgcrd",
"jsonPayload": {
"repository_storage": "default",
"meta.client_id": "ip/34.65.234.178",
"db_main_replica_txn_max_duration_s": 0,
"redis_feature_flag_write_bytes": 53,
"rate_limiting_gates": [],
"component": "gitlab",
"db_main_replica_txn_duration_s": 0,
"db_primary_txn_count": 0,
"target_duration_s": 1,
"db_ci_txn_max_duration_s": 0,
"db_replica_cached_count": 1,
"db_main_txn_duration_s": 0,
"db_ci_txn_count": 0,
"view_duration_s": 0.00023,
"db_main_duration_s": 0,
"redis_feature_flag_read_bytes": 199,
"controller": "Repositories::GitHttpController",
"redis_feature_flag_calls": 1,
"db_ci_wal_cached_count": 0,
"meta.remote_ip": "34.65.234.178",
"redis_calls": 2,
"db_replica_duration_s": 0.009,
"redis_repository_cache_read_bytes": 101,
"db_txn_count": 0,
"db_ci_replica_txn_max_duration_s": 0,
"db_ci_cached_count": 0,
"redis_duration_s": 0.0007830000000000001,
"db_main_wal_count": 0,
"db_main_txn_max_duration_s": 0,
"cpu_s": 0.039023,
"db_ci_replica_count": 0,
"db_count": 11,
"db_primary_cached_count": 0,
"db_main_replica_cached_count": 1,
"pid": 380,
"db_main_wal_cached_count": 0,
"redis_read_bytes": 300,
"db_write_count": 0,
"redis_feature_flag_duration_s": 0.00036,
"db_duration_s": 0.00774,
"db_main_replica_duration_s": 0.009,
"status": 200,
"db_ci_replica_cached_count": 0,
"mem_mallocs": 2979,
"db_replica_txn_max_duration_s": 0,
"db_main_replica_count": 11,
"db_replica_txn_count": 0,
"params": [
{
"key": "service",
"value": "git-upload-pack"
},
{
"value": "mkozonotest/gitlab-runner.git",
"key": "repository_path"
}
],
"redis_repository_cache_calls": 1,
"db_primary_txn_duration_s": 0,
"db_replica_txn_duration_s": 0,
"mem_total_bytes": 1346128,
"db_ci_count": 0,
"db_ci_wal_count": 0,
"redis_repository_cache_duration_s": 0.000423,
"db_primary_wal_cached_count": 0,
"redis_repository_cache_write_bytes": 55,
"meta.root_namespace": "mkozonotest",
"db_ci_replica_txn_count": 0,
"duration_s": 0.03681,
"db_main_replica_wal_cached_count": 0,
"db_ci_duration_s": 0,
"db_ci_replica_duration_s": 0,
"db_ci_replica_wal_cached_count": 0,
"db_main_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"mem_objects": 13070,
"db_primary_count": 0,
"meta.project": "mkozonotest/gitlab-runner",
"db_primary_wal_count": 0,
"worker_id": "puma_3",
"db_main_cached_count": 0,
"remote_ip": "34.65.234.178",
"subcomponent": "production_json",
"mem_bytes": 823328,
"format": "*/*",
"path": "/mkozonotest/gitlab-runner.git/info/refs",
"db_primary_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"db_cached_count": 1,
"request_urgency": "default",
"db_main_replica_txn_count": 0,
"db_main_txn_count": 0,
"db_ci_replica_txn_duration_s": 0,
"db_ci_replica_wal_count": 0,
"redis_write_bytes": 108,
"meta.feature_category": "source_code_management",
"action": "info_refs",
"ua": "git/2.43.2",
"db_main_count": 0,
"meta.caller_id": "Repositories::GitHttpController#info_refs",
"method": "GET",
"db_ci_txn_duration_s": 0,
"db_replica_wal_count": 0,
"db_replica_count": 11,
"correlation_id": "01HTQV6DHSTQDCGA9AV26TDKRB"
},
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "webservice",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"namespace_name": "default",
"location": "us-east1-c"
}
},
"timestamp": "2024-04-05T19:19:01.482Z",
"severity": "ERROR",
"labels": {
"k8s-pod/shard": "default",
"k8s-pod/app": "webservice",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/stage": "main",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/heritage": "Helm",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/type": "web",
"k8s-pod/tier": "sv",
"k8s-pod/release": "gitlab",
"k8s-pod/chart": "webservice-7.10.1"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:05.896021723Z"
},
{
"textPayload": "34.65.5.0 - - [05/Apr/2024:19:19:01 +0000] \"GET /mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack HTTP/1.1\" 200 160 \"-\" \"git/2.43.2\" 547 0.069 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 160 0.069 200 0719f4cb480ffb9ed4f75df4b2c51181",
"insertId": "dkarajtkcfgfmwgh",
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth",
"project_id": "gitlab-staging-ref",
"location": "us-east1-c",
"container_name": "controller",
"cluster_name": "staging-ref-3k-hybrid-us"
}
},
"timestamp": "2024-04-05T19:19:01.485820718Z",
"severity": "INFO",
"labels": {
"k8s-pod/stage": "main",
"k8s-pod/release": "gitlab",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/component": "controller",
"k8s-pod/type": "nginx",
"k8s-pod/shard": "default",
"k8s-pod/app": "nginx-ingress",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.161949943Z"
},
{
"textPayload": "34.65.122.187 - - [05/Apr/2024:19:19:01 +0000] \"GET /mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack HTTP/1.1\" 200 160 \"-\" \"git/2.43.2\" 547 0.070 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 160 0.070 200 8bb23191479d66931ae9b73e7fdb7bf3",
"insertId": "nbf1cqnhyhwoaa7d",
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "controller",
"cluster_name": "staging-ref-3k-hybrid-us",
"project_id": "gitlab-staging-ref",
"namespace_name": "default",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth",
"location": "us-east1-c"
}
},
"timestamp": "2024-04-05T19:19:01.485964927Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/shard": "default",
"k8s-pod/component": "controller",
"k8s-pod/stage": "main",
"k8s-pod/type": "nginx",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/app": "nginx-ingress"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.161949943Z"
},
{
"textPayload": "34.65.234.178 - - [05/Apr/2024:19:19:01 +0000] \"GET /mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack HTTP/1.1\" 200 160 \"-\" \"git/2.43.2\" 547 0.064 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 160 0.064 200 015b7cc35dd5f3f73e25947457ea9f12",
"insertId": "m269alteqaa2tnjy",
"resource": {
"type": "k8s_container",
"labels": {
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-cvvjr",
"namespace_name": "default",
"container_name": "controller",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:01.496774177Z",
"severity": "INFO",
"labels": {
"k8s-pod/app": "nginx-ingress",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/release": "gitlab",
"k8s-pod/component": "controller",
"k8s-pod/type": "nginx",
"k8s-pod/shard": "default",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/stage": "main"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.159041917Z"
},
{
"insertId": "y2yny5e2fh6vhv9e",
"jsonPayload": {
"mem_objects": 15191,
"db_main_cached_count": 0,
"view_duration_s": 0.00024,
"redis_feature_flag_calls": 4,
"db_ci_cached_count": 0,
"correlation_id": "01HTQV6DPNKA1GXFP36EPHJMSC",
"db_main_txn_count": 0,
"db_ci_replica_count": 0,
"db_primary_wal_cached_count": 0,
"duration_s": 0.04822,
"db_main_replica_wal_cached_count": 0,
"controller": "Repositories::GitHttpController",
"db_duration_s": 0.01336,
"db_main_duration_s": 0.003,
"component": "gitlab",
"db_replica_txn_count": 0,
"db_replica_txn_max_duration_s": 0,
"redis_repository_cache_write_bytes": 55,
"db_ci_txn_max_duration_s": 0,
"pid": 1077,
"db_main_replica_txn_count": 0,
"meta.remote_ip": "34.65.122.187",
"db_primary_txn_duration_s": 0,
"meta.root_namespace": "mkozonotest",
"db_txn_count": 0,
"subcomponent": "production_json",
"db_primary_txn_count": 0,
"db_ci_replica_txn_count": 0,
"ua": "git/2.43.2",
"method": "POST",
"db_ci_wal_count": 0,
"db_replica_txn_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"rate_limiting_gates": [],
"db_main_replica_txn_max_duration_s": 0,
"db_ci_count": 0,
"redis_repository_cache_duration_s": 0.000483,
"db_main_replica_duration_s": 0.01,
"db_ci_replica_duration_s": 0,
"db_ci_replica_wal_cached_count": 0,
"db_write_count": 1,
"target_duration_s": 1,
"db_primary_cached_count": 0,
"db_count": 13,
"meta.client_id": "ip/34.65.122.187",
"redis_repository_cache_calls": 1,
"db_main_txn_max_duration_s": 0,
"remote_ip": "34.65.122.187",
"db_main_replica_count": 12,
"db_ci_txn_count": 0,
"db_primary_duration_s": 0.003,
"redis_duration_s": 0.002066,
"redis_repository_cache_read_bytes": 101,
"db_primary_wal_count": 0,
"db_main_wal_cached_count": 0,
"db_replica_count": 12,
"db_ci_txn_duration_s": 0,
"meta.feature_category": "source_code_management",
"db_cached_count": 1,
"meta.project": "mkozonotest/gitlab-runner",
"db_replica_duration_s": 0.01,
"db_ci_replica_txn_duration_s": 0,
"cpu_s": 0.044266,
"mem_total_bytes": 1463200,
"redis_calls": 5,
"db_ci_replica_cached_count": 0,
"path": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"db_ci_wal_cached_count": 0,
"params": [
{
"value": "mkozonotest/gitlab-runner.git",
"key": "repository_path"
}
],
"action": "git_upload_pack",
"redis_feature_flag_duration_s": 0.001583,
"db_ci_replica_txn_max_duration_s": 0,
"request_urgency": "default",
"redis_feature_flag_write_bytes": 454,
"repository_storage": "default",
"db_main_replica_wal_count": 0,
"db_main_txn_duration_s": 0,
"status": 200,
"db_main_replica_txn_duration_s": 0,
"db_main_wal_count": 0,
"redis_read_bytes": 500,
"format": null,
"worker_id": "puma_1",
"mem_mallocs": 3296,
"db_ci_duration_s": 0,
"db_replica_cached_count": 1,
"db_main_count": 1,
"db_primary_count": 1,
"meta.caller_id": "Repositories::GitHttpController#git_upload_pack",
"redis_feature_flag_read_bytes": 399,
"db_main_replica_cached_count": 1,
"redis_write_bytes": 509,
"mem_bytes": 855560,
"db_ci_replica_wal_count": 0,
"db_replica_wal_count": 0,
"db_replica_wal_cached_count": 0
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "webservice",
"project_id": "gitlab-staging-ref",
"location": "us-east1-c",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx"
}
},
"timestamp": "2024-04-05T19:19:01.650Z",
"severity": "ERROR",
"labels": {
"k8s-pod/stage": "main",
"k8s-pod/tier": "sv",
"k8s-pod/release": "gitlab",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/heritage": "Helm",
"k8s-pod/shard": "default",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/app": "webservice",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/type": "web",
"k8s-pod/gitlab_com/webservice-name": "default"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:05.896021723Z"
},
{
"insertId": "73t478y0xqmu9ew7",
"jsonPayload": {
"meta.remote_ip": "34.65.234.178",
"db_main_duration_s": 0.002,
"view_duration_s": 0.00028,
"redis_feature_flag_write_bytes": 129,
"db_ci_wal_count": 0,
"meta.feature_category": "source_code_management",
"db_ci_txn_count": 0,
"params": [
{
"value": "mkozonotest/gitlab-runner.git",
"key": "repository_path"
}
],
"db_replica_cached_count": 1,
"db_ci_txn_max_duration_s": 0,
"db_replica_txn_max_duration_s": 0,
"db_main_replica_txn_count": 0,
"db_primary_txn_duration_s": 0,
"db_ci_cached_count": 0,
"meta.client_id": "ip/34.65.234.178",
"db_ci_replica_txn_max_duration_s": 0,
"db_ci_txn_duration_s": 0,
"pid": 469,
"redis_duration_s": 0.001024,
"db_count": 12,
"db_replica_wal_cached_count": 0,
"path": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"target_duration_s": 1,
"db_ci_replica_txn_duration_s": 0,
"db_primary_cached_count": 0,
"db_main_wal_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_ci_replica_duration_s": 0,
"db_write_count": 1,
"meta.caller_id": "Repositories::GitHttpController#git_upload_pack",
"db_ci_wal_cached_count": 0,
"db_main_replica_cached_count": 1,
"redis_repository_cache_calls": 1,
"db_primary_count": 1,
"db_replica_duration_s": 0.009,
"meta.root_namespace": "mkozonotest",
"db_duration_s": 0.01088,
"method": "POST",
"db_replica_count": 11,
"db_primary_duration_s": 0.002,
"db_primary_wal_cached_count": 0,
"duration_s": 0.04064,
"cpu_s": 0.040024,
"db_ci_replica_cached_count": 0,
"db_ci_count": 0,
"status": 200,
"db_main_replica_txn_duration_s": 0,
"db_main_replica_count": 11,
"redis_feature_flag_read_bytes": 397,
"redis_write_bytes": 184,
"db_ci_replica_wal_count": 0,
"ua": "git/2.43.2",
"redis_feature_flag_calls": 2,
"db_main_replica_duration_s": 0.009,
"subcomponent": "production_json",
"worker_id": "puma_2",
"db_replica_txn_count": 0,
"mem_mallocs": 3163,
"redis_repository_cache_write_bytes": 55,
"redis_repository_cache_read_bytes": 101,
"format": null,
"action": "git_upload_pack",
"db_main_replica_txn_max_duration_s": 0,
"repository_storage": "default",
"mem_bytes": 839344,
"db_main_txn_duration_s": 0,
"db_main_txn_max_duration_s": 0,
"meta.project": "mkozonotest/gitlab-runner",
"db_ci_replica_count": 0,
"db_main_replica_wal_cached_count": 0,
"mem_objects": 14150,
"db_main_cached_count": 0,
"db_primary_wal_count": 0,
"db_cached_count": 1,
"db_main_replica_wal_count": 0,
"db_main_count": 1,
"controller": "Repositories::GitHttpController",
"db_main_wal_cached_count": 0,
"db_primary_txn_max_duration_s": 0,
"db_main_txn_count": 0,
"component": "gitlab",
"remote_ip": "34.65.234.178",
"db_ci_replica_txn_count": 0,
"redis_calls": 3,
"request_urgency": "default",
"redis_read_bytes": 498,
"mem_total_bytes": 1405344,
"correlation_id": "01HTQV6DQ0K7KSHFVR4KC8D5C2",
"rate_limiting_gates": [],
"redis_repository_cache_duration_s": 0.000358,
"db_txn_count": 0,
"redis_feature_flag_duration_s": 0.000666,
"db_ci_duration_s": 0,
"db_replica_txn_duration_s": 0,
"db_primary_txn_count": 0,
"db_replica_wal_count": 0
},
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-us",
"namespace_name": "default",
"container_name": "webservice",
"location": "us-east1-c"
}
},
"timestamp": "2024-04-05T19:19:01.652Z",
"severity": "ERROR",
"labels": {
"k8s-pod/gitlab_com/webservice-name": "default",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/heritage": "Helm",
"k8s-pod/type": "web",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/tier": "sv",
"k8s-pod/shard": "default",
"k8s-pod/release": "gitlab",
"k8s-pod/app": "webservice",
"k8s-pod/stage": "main",
"k8s-pod/chart": "webservice-7.10.1"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:05.896021723Z"
},
{
"insertId": "tvbeq7iimuvul11l",
"jsonPayload": {
"target_duration_s": 1,
"meta.remote_ip": "34.65.5.0",
"db_primary_count": 1,
"action": "git_upload_pack",
"view_duration_s": 0.00028,
"db_primary_txn_count": 0,
"db_ci_wal_count": 0,
"worker_id": "puma_3",
"db_duration_s": 0.01839,
"component": "gitlab",
"redis_repository_cache_write_bytes": 55,
"db_replica_txn_count": 0,
"db_ci_replica_txn_max_duration_s": 0,
"meta.feature_category": "source_code_management",
"db_ci_replica_txn_count": 0,
"method": "POST",
"db_main_txn_duration_s": 0,
"path": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"meta.caller_id": "Repositories::GitHttpController#git_upload_pack",
"db_main_txn_count": 0,
"meta.client_id": "ip/34.65.5.0",
"db_ci_replica_wal_count": 0,
"db_ci_replica_txn_duration_s": 0,
"db_main_duration_s": 0.002,
"mem_total_bytes": 1502648,
"remote_ip": "34.65.5.0",
"db_primary_cached_count": 0,
"redis_read_bytes": 1454,
"db_main_count": 1,
"db_main_replica_wal_cached_count": 0,
"db_ci_txn_max_duration_s": 0,
"db_primary_wal_count": 0,
"correlation_id": "01HTQV6DPNQ760JPFYCN96G41P",
"ua": "git/2.43.2",
"db_ci_replica_count": 0,
"db_main_replica_txn_count": 0,
"db_ci_replica_cached_count": 0,
"db_replica_txn_max_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"mem_bytes": 911568,
"db_txn_count": 0,
"params": [
{
"key": "repository_path",
"value": "mkozonotest/gitlab-runner.git"
}
],
"db_count": 12,
"pid": 508,
"mem_objects": 14777,
"db_replica_txn_duration_s": 0,
"db_replica_cached_count": 1,
"db_ci_txn_count": 0,
"redis_repository_cache_calls": 1,
"db_main_replica_wal_count": 0,
"db_cached_count": 1,
"rate_limiting_gates": [],
"duration_s": 0.06139,
"db_main_wal_count": 0,
"redis_feature_flag_read_bytes": 1353,
"db_main_txn_max_duration_s": 0,
"status": 200,
"db_main_replica_txn_max_duration_s": 0,
"db_ci_replica_duration_s": 0,
"db_main_replica_duration_s": 0.016,
"db_ci_count": 0,
"mem_mallocs": 3375,
"db_main_replica_txn_duration_s": 0,
"redis_repository_cache_read_bytes": 101,
"db_primary_txn_duration_s": 0,
"db_write_count": 1,
"request_urgency": "default",
"meta.root_namespace": "mkozonotest",
"repository_storage": "default",
"db_replica_wal_count": 0,
"db_main_cached_count": 0,
"controller": "Repositories::GitHttpController",
"db_replica_count": 11,
"redis_repository_cache_duration_s": 0.000588,
"db_main_replica_cached_count": 1,
"db_ci_txn_duration_s": 0,
"db_primary_wal_cached_count": 0,
"meta.project": "mkozonotest/gitlab-runner",
"db_main_replica_count": 11,
"cpu_s": 0.053724,
"db_ci_duration_s": 0,
"db_replica_wal_cached_count": 0,
"redis_feature_flag_write_bytes": 217,
"redis_duration_s": 0.002773,
"db_ci_cached_count": 0,
"db_main_wal_cached_count": 0,
"subcomponent": "production_json",
"redis_feature_flag_duration_s": 0.002185,
"format": null,
"redis_write_bytes": 272,
"db_ci_replica_wal_cached_count": 0,
"db_replica_duration_s": 0.016,
"db_primary_duration_s": 0.002,
"redis_feature_flag_calls": 4,
"db_ci_wal_cached_count": 0,
"redis_calls": 5
},
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-webservice-default-669cf4599-qww8d",
"container_name": "webservice",
"project_id": "gitlab-staging-ref",
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-us",
"location": "us-east1-c"
}
},
"timestamp": "2024-04-05T19:19:01.664Z",
"severity": "ERROR",
"labels": {
"k8s-pod/stage": "main",
"k8s-pod/tier": "sv",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/release": "gitlab",
"k8s-pod/app": "webservice",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/shard": "default",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/type": "web",
"k8s-pod/heritage": "Helm"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:04.927265209Z"
},
{
"textPayload": "34.65.122.187 - - [05/Apr/2024:19:19:01 +0000] \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/1.1\" 200 113849 \"-\" \"git/2.43.2\" 768 0.179 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 113662 0.179 200 c686f6f7750c47d849ede659f5f56475",
"insertId": "67t7fzxeil6cy9jz",
"resource": {
"type": "k8s_container",
"labels": {
"location": "us-east1-c",
"container_name": "controller",
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-us",
"namespace_name": "default",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth"
}
},
"timestamp": "2024-04-05T19:19:01.769241920Z",
"severity": "INFO",
"labels": {
"k8s-pod/shard": "default",
"k8s-pod/app": "nginx-ingress",
"k8s-pod/release": "gitlab",
"k8s-pod/pod-template-hash": "6988b54ffb",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/component": "controller",
"k8s-pod/type": "nginx",
"k8s-pod/stage": "main"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.161949943Z"
},
{
"textPayload": "34.65.234.178 - - [05/Apr/2024:19:19:01 +0000] \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/1.1\" 200 113849 \"-\" \"git/2.43.2\" 768 0.170 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 113662 0.169 200 b1fb61a42a4c3305232e394680ef5b81",
"insertId": "0gokt2uptj8rjos6",
"resource": {
"type": "k8s_container",
"labels": {
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "controller",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-cvvjr",
"namespace_name": "default",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:01.770335847Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/type": "nginx",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/release": "gitlab",
"k8s-pod/component": "controller",
"k8s-pod/stage": "main",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/app": "nginx-ingress",
"k8s-pod/shard": "default"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.159041917Z"
},
{
"textPayload": "34.65.5.0 - - [05/Apr/2024:19:19:01 +0000] \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/1.1\" 200 113856 \"-\" \"git/2.43.2\" 768 0.194 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 113662 0.193 200 40365ed87442468a25efe00e62240cd9",
"insertId": "9t9y50u0ca2abf0x",
"resource": {
"type": "k8s_container",
"labels": {
"cluster_name": "staging-ref-3k-hybrid-us",
"location": "us-east1-c",
"project_id": "gitlab-staging-ref",
"container_name": "controller",
"namespace_name": "default",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth"
}
},
"timestamp": "2024-04-05T19:19:01.783070672Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/app": "nginx-ingress",
"k8s-pod/release": "gitlab",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/component": "controller",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/shard": "default",
"k8s-pod/type": "nginx",
"k8s-pod/stage": "main"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.161949943Z"
},
{
"insertId": "3xe02e3q4ogfneyo",
"jsonPayload": {
"msg": "access",
"level": "info",
"duration_ms": 126,
"method": "POST",
"content_type": "application/x-git-upload-pack-result",
"remote_ip": "34.65.5.0",
"correlation_id": "01HTQV6E7151816GFVD0XTA3MD",
"proto": "HTTP/1.1",
"referrer": "",
"uri": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"written_bytes": 1173,
"user_agent": "git/2.43.2",
"ttfb_ms": 76,
"status": 200,
"route": "^/.+\\.git/git-upload-pack\\z",
"remote_addr": "34.65.5.0:58606",
"host": "staging-ref.gitlab.com",
"system": "http"
},
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "gitlab-workhorse",
"pod_name": "gitlab-webservice-default-669cf4599-qww8d",
"namespace_name": "default",
"project_id": "gitlab-staging-ref",
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us"
}
},
"timestamp": "2024-04-05T19:19:02Z",
"severity": "INFO",
"labels": {
"k8s-pod/type": "web",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/release": "gitlab",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/app": "webservice",
"k8s-pod/heritage": "Helm",
"k8s-pod/tier": "sv",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/stage": "main",
"k8s-pod/shard": "default",
"k8s-pod/chart": "webservice-7.10.1"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.930552910Z"
},
{
"insertId": "405ta6r2b9onh3a9",
"jsonPayload": {
"remote_addr": "34.65.122.187:36968",
"correlation_id": "01HTQV6E6MR82Z0VHBWKWNQKEC",
"system": "http",
"msg": "access",
"uri": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"content_type": "application/x-git-upload-pack-result",
"written_bytes": 1173,
"host": "staging-ref.gitlab.com",
"duration_ms": 116,
"user_agent": "git/2.43.2",
"proto": "HTTP/1.1",
"status": 200,
"referrer": "",
"method": "POST",
"level": "info",
"route": "^/.+\\.git/git-upload-pack\\z",
"ttfb_ms": 65,
"remote_ip": "34.65.122.187"
},
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"container_name": "gitlab-workhorse",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"namespace_name": "default",
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us"
}
},
"timestamp": "2024-04-05T19:19:02Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/shard": "default",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/heritage": "Helm",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/app": "webservice",
"k8s-pod/type": "web",
"k8s-pod/tier": "sv",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/stage": "main",
"k8s-pod/pod-template-hash": "669cf4599"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:05.890135794Z"
},
{
"insertId": "93ieo1mstq6mhtix",
"jsonPayload": {
"user_agent": "git/2.43.2",
"msg": "access",
"duration_ms": 160,
"written_bytes": 4225,
"uri": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"remote_ip": "34.65.234.178",
"proto": "HTTP/1.1",
"level": "info",
"content_type": "application/x-git-upload-pack-result",
"system": "http",
"remote_addr": "34.65.234.178:45134",
"method": "POST",
"status": 200,
"route": "^/.+\\.git/git-upload-pack\\z",
"host": "staging-ref.gitlab.com",
"referrer": "",
"ttfb_ms": 68,
"correlation_id": "01HTQV6E6NFNAKS9EN329FWK3D"
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"cluster_name": "staging-ref-3k-hybrid-us",
"location": "us-east1-c",
"project_id": "gitlab-staging-ref",
"container_name": "gitlab-workhorse"
}
},
"timestamp": "2024-04-05T19:19:02Z",
"severity": "INFO",
"labels": {
"k8s-pod/stage": "main",
"k8s-pod/app": "webservice",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/shard": "default",
"k8s-pod/type": "web",
"k8s-pod/release": "gitlab",
"k8s-pod/heritage": "Helm",
"k8s-pod/tier": "sv",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/pod-template-hash": "669cf4599"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:05.890135794Z"
},
{
"insertId": "dbhwto20fv7tdkke",
"jsonPayload": {
"referrer": "",
"correlation_id": "01HTQV6EYA1MNZQ7DJ3CZ66M7V",
"content_type": "application/x-git-upload-pack-advertisement",
"remote_ip": "34.65.5.0",
"level": "info",
"written_bytes": 160,
"duration_ms": 70,
"msg": "access",
"user_agent": "git/2.43.2",
"remote_addr": "34.65.5.0:58606",
"method": "GET",
"route": "^/.+\\.git/info/refs\\z",
"host": "staging-ref.gitlab.com",
"uri": "/mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack",
"system": "http",
"status": 200,
"proto": "HTTP/1.1",
"ttfb_ms": 64
},
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "gitlab-workhorse",
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-webservice-default-669cf4599-qww8d",
"cluster_name": "staging-ref-3k-hybrid-us",
"namespace_name": "default",
"location": "us-east1-c"
}
},
"timestamp": "2024-04-05T19:19:02Z",
"severity": "INFO",
"labels": {
"k8s-pod/shard": "default",
"k8s-pod/app": "webservice",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/release": "gitlab",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/chart": "webservice-7.10.1",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/tier": "sv",
"k8s-pod/stage": "main",
"k8s-pod/heritage": "Helm",
"k8s-pod/type": "web"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.930552910Z"
},
{
"insertId": "7shd0vrmoebwdt5x",
"jsonPayload": {
"db_primary_txn_count": 0,
"db_main_txn_count": 0,
"correlation_id": "01HTQV6E6MR82Z0VHBWKWNQKEC",
"db_replica_txn_duration_s": 0,
"duration_s": 0.03903,
"db_primary_txn_duration_s": 0,
"db_ci_wal_count": 0,
"db_replica_cached_count": 1,
"db_main_count": 1,
"db_primary_wal_cached_count": 0,
"meta.feature_category": "source_code_management",
"db_ci_replica_txn_duration_s": 0,
"params": [
{
"key": "repository_path",
"value": "mkozonotest/gitlab-runner.git"
}
],
"view_duration_s": 0.00022,
"action": "git_upload_pack",
"db_primary_count": 1,
"pid": 1077,
"db_ci_replica_txn_max_duration_s": 0,
"request_urgency": "default",
"db_main_replica_count": 11,
"db_main_txn_duration_s": 0,
"ua": "git/2.43.2",
"repository_storage": "default",
"controller": "Repositories::GitHttpController",
"meta.client_id": "ip/34.65.122.187",
"db_main_replica_wal_count": 0,
"db_main_wal_count": 0,
"db_ci_replica_txn_count": 0,
"meta.remote_ip": "34.65.122.187",
"subcomponent": "production_json",
"rate_limiting_gates": [],
"db_replica_duration_s": 0.009,
"db_ci_replica_wal_count": 0,
"db_ci_txn_max_duration_s": 0,
"db_ci_replica_duration_s": 0,
"mem_bytes": 820944,
"format": null,
"db_ci_duration_s": 0,
"db_main_txn_max_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"db_main_replica_txn_duration_s": 0,
"db_main_replica_txn_max_duration_s": 0,
"db_primary_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"mem_total_bytes": 1367264,
"db_ci_cached_count": 0,
"db_duration_s": 0.01121,
"db_ci_wal_cached_count": 0,
"db_main_replica_txn_count": 0,
"db_write_count": 1,
"db_replica_txn_count": 0,
"db_main_duration_s": 0.003,
"db_replica_wal_count": 0,
"db_ci_txn_duration_s": 0,
"db_ci_txn_count": 0,
"component": "gitlab",
"redis_repository_cache_duration_s": 0.000566,
"redis_duration_s": 0.000566,
"meta.project": "mkozonotest/gitlab-runner",
"redis_calls": 1,
"redis_repository_cache_calls": 1,
"db_ci_replica_cached_count": 0,
"db_txn_count": 0,
"db_ci_count": 0,
"status": 200,
"worker_id": "puma_1",
"redis_repository_cache_write_bytes": 55,
"meta.caller_id": "Repositories::GitHttpController#git_upload_pack",
"mem_mallocs": 3019,
"db_main_replica_wal_cached_count": 0,
"meta.root_namespace": "mkozonotest",
"redis_write_bytes": 55,
"target_duration_s": 1,
"mem_objects": 13658,
"method": "POST",
"redis_repository_cache_read_bytes": 101,
"cpu_s": 0.037858,
"db_replica_wal_cached_count": 0,
"db_replica_count": 11,
"remote_ip": "34.65.122.187",
"db_main_replica_cached_count": 1,
"db_replica_txn_max_duration_s": 0,
"db_main_wal_cached_count": 0,
"db_count": 12,
"db_main_cached_count": 0,
"db_primary_duration_s": 0.003,
"db_cached_count": 1,
"path": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"db_ci_replica_count": 0,
"redis_read_bytes": 101,
"db_primary_wal_count": 0,
"db_main_replica_duration_s": 0.009
},
"resource": {
"type": "k8s_container",
"labels": {
"location": "us-east1-c",
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "webservice",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:02.153Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/tier": "sv",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/stage": "main",
"k8s-pod/app": "webservice",
"k8s-pod/release": "gitlab",
"k8s-pod/heritage": "Helm",
"k8s-pod/shard": "default",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/type": "web"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:05.896021723Z"
},
{
"insertId": "noqd8l0154pvmkyf",
"jsonPayload": {
"redis_feature_flag_calls": 3,
"db_txn_count": 0,
"redis_repository_cache_duration_s": 0.000389,
"db_ci_duration_s": 0,
"db_replica_wal_count": 0,
"worker_id": "puma_0",
"db_ci_txn_count": 0,
"db_primary_txn_duration_s": 0,
"db_ci_cached_count": 0,
"mem_total_bytes": 1424632,
"db_primary_wal_count": 0,
"db_ci_replica_txn_count": 0,
"meta.root_namespace": "mkozonotest",
"redis_repository_cache_read_bytes": 101,
"meta.caller_id": "Repositories::GitHttpController#git_upload_pack",
"method": "POST",
"redis_write_bytes": 237,
"component": "gitlab",
"db_main_replica_cached_count": 1,
"db_main_duration_s": 0.003,
"subcomponent": "production_json",
"request_urgency": "default",
"mem_bytes": 847312,
"target_duration_s": 1,
"db_main_cached_count": 0,
"db_main_txn_max_duration_s": 0,
"db_primary_txn_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_count": 0,
"action": "git_upload_pack",
"mem_mallocs": 3235,
"redis_calls": 4,
"db_count": 12,
"db_main_replica_txn_count": 0,
"db_ci_txn_max_duration_s": 0,
"db_ci_replica_duration_s": 0,
"db_ci_replica_wal_cached_count": 0,
"db_cached_count": 1,
"db_main_replica_txn_max_duration_s": 0,
"db_ci_replica_wal_count": 0,
"db_ci_replica_txn_max_duration_s": 0,
"db_main_wal_count": 0,
"db_ci_replica_cached_count": 0,
"db_main_replica_txn_duration_s": 0,
"ua": "git/2.43.2",
"db_write_count": 1,
"db_replica_cached_count": 1,
"pid": 400,
"db_main_replica_duration_s": 0.008,
"controller": "Repositories::GitHttpController",
"meta.client_id": "ip/34.65.234.178",
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_count": 0,
"cpu_s": 0.039941,
"db_main_count": 1,
"redis_feature_flag_write_bytes": 182,
"redis_feature_flag_duration_s": 0.001302,
"db_primary_wal_cached_count": 0,
"db_ci_wal_count": 0,
"rate_limiting_gates": [],
"repository_storage": "default",
"db_main_replica_count": 11,
"redis_feature_flag_read_bytes": 596,
"remote_ip": "34.65.234.178",
"duration_s": 0.04171,
"correlation_id": "01HTQV6E6NFNAKS9EN329FWK3D",
"db_primary_txn_max_duration_s": 0,
"redis_repository_cache_calls": 1,
"db_ci_replica_txn_duration_s": 0,
"db_primary_duration_s": 0.003,
"db_duration_s": 0.01059,
"db_ci_txn_duration_s": 0,
"meta.remote_ip": "34.65.234.178",
"redis_read_bytes": 697,
"db_replica_duration_s": 0.008,
"db_primary_count": 1,
"meta.feature_category": "source_code_management",
"db_replica_txn_duration_s": 0,
"redis_repository_cache_write_bytes": 55,
"db_ci_count": 0,
"db_primary_cached_count": 0,
"params": [
{
"value": "mkozonotest/gitlab-runner.git",
"key": "repository_path"
}
],
"mem_objects": 14433,
"meta.project": "mkozonotest/gitlab-runner",
"status": 200,
"db_replica_count": 11,
"redis_duration_s": 0.001691,
"db_main_txn_duration_s": 0,
"db_replica_wal_cached_count": 0,
"db_main_txn_count": 0,
"db_main_wal_cached_count": 0,
"format": null,
"db_replica_txn_count": 0,
"db_replica_txn_max_duration_s": 0,
"path": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"view_duration_s": 0.00025
},
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "webservice",
"project_id": "gitlab-staging-ref",
"namespace_name": "default",
"location": "us-east1-c"
}
},
"timestamp": "2024-04-05T19:19:02.155Z",
"severity": "ERROR",
"labels": {
"k8s-pod/app": "webservice",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/tier": "sv",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/type": "web",
"k8s-pod/stage": "main",
"k8s-pod/heritage": "Helm",
"k8s-pod/shard": "default",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/release": "gitlab",
"k8s-pod/pod-template-hash": "669cf4599"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:05.896021723Z"
},
{
"insertId": "10rx6omldh6p9b50",
"jsonPayload": {
"db_main_replica_duration_s": 0.01,
"db_ci_duration_s": 0,
"db_count": 12,
"duration_s": 0.04895,
"db_main_replica_txn_max_duration_s": 0,
"redis_calls": 2,
"action": "git_upload_pack",
"db_main_replica_count": 11,
"db_main_txn_max_duration_s": 0,
"db_ci_wal_count": 0,
"db_main_wal_cached_count": 0,
"redis_feature_flag_read_bytes": 199,
"meta.caller_id": "Repositories::GitHttpController#git_upload_pack",
"redis_repository_cache_write_bytes": 55,
"mem_bytes": 832032,
"db_ci_wal_cached_count": 0,
"component": "gitlab",
"db_main_txn_duration_s": 0,
"redis_feature_flag_write_bytes": 64,
"meta.feature_category": "source_code_management",
"db_primary_txn_count": 0,
"pid": 529,
"db_main_replica_txn_count": 0,
"redis_feature_flag_calls": 1,
"db_primary_wal_count": 0,
"db_main_replica_txn_duration_s": 0,
"db_cached_count": 1,
"status": 200,
"db_txn_count": 0,
"db_replica_wal_count": 0,
"db_ci_replica_count": 0,
"controller": "Repositories::GitHttpController",
"format": null,
"db_main_count": 1,
"redis_repository_cache_read_bytes": 101,
"db_ci_txn_max_duration_s": 0,
"db_main_replica_wal_count": 0,
"redis_feature_flag_duration_s": 0.000438,
"db_replica_txn_duration_s": 0,
"request_urgency": "default",
"db_primary_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"mem_mallocs": 3100,
"db_replica_duration_s": 0.01,
"db_main_wal_count": 0,
"db_ci_replica_txn_count": 0,
"view_duration_s": 0.00026,
"db_primary_wal_cached_count": 0,
"correlation_id": "01HTQV6E7151816GFVD0XTA3MD",
"db_ci_count": 0,
"db_primary_duration_s": 0.003,
"db_ci_replica_duration_s": 0,
"remote_ip": "34.65.5.0",
"db_replica_txn_count": 0,
"db_main_replica_cached_count": 1,
"worker_id": "puma_2",
"repository_storage": "default",
"db_replica_txn_max_duration_s": 0,
"db_main_txn_count": 0,
"db_replica_count": 11,
"redis_duration_s": 0.001118,
"redis_read_bytes": 300,
"db_ci_txn_duration_s": 0,
"db_ci_replica_cached_count": 0,
"path": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"db_main_cached_count": 0,
"meta.client_id": "ip/34.65.5.0",
"params": [
{
"key": "repository_path",
"value": "mkozonotest/gitlab-runner.git"
}
],
"mem_objects": 13941,
"redis_repository_cache_duration_s": 0.00068,
"mem_total_bytes": 1389672,
"db_ci_cached_count": 0,
"db_primary_txn_max_duration_s": 0,
"db_main_duration_s": 0.003,
"ua": "git/2.43.2",
"subcomponent": "production_json",
"db_duration_s": 0.01255,
"db_primary_count": 1,
"meta.project": "mkozonotest/gitlab-runner",
"db_ci_replica_txn_max_duration_s": 0,
"db_write_count": 1,
"db_ci_replica_wal_count": 0,
"redis_repository_cache_calls": 1,
"meta.remote_ip": "34.65.5.0",
"db_ci_replica_txn_duration_s": 0,
"db_ci_replica_wal_cached_count": 0,
"db_primary_txn_duration_s": 0,
"rate_limiting_gates": [],
"db_replica_cached_count": 1,
"meta.root_namespace": "mkozonotest",
"redis_write_bytes": 119,
"cpu_s": 0.047502,
"target_duration_s": 1,
"db_ci_txn_count": 0,
"method": "POST",
"db_replica_wal_cached_count": 0
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-us",
"location": "us-east1-c",
"container_name": "webservice",
"pod_name": "gitlab-webservice-default-669cf4599-qww8d"
}
},
"timestamp": "2024-04-05T19:19:02.176Z",
"severity": "ERROR",
"labels": {
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/type": "web",
"k8s-pod/tier": "sv",
"k8s-pod/stage": "main",
"k8s-pod/app": "webservice",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/heritage": "Helm",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/release": "gitlab",
"k8s-pod/shard": "default"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:04.927265209Z"
},
{
"textPayload": "34.65.122.187 - - [05/Apr/2024:19:19:02 +0000] \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/1.1\" 200 1173 \"-\" \"git/2.43.2\" 1577 0.118 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 1173 0.118 200 9ae6b5e9d4ce690acfeb83a59ef98c09",
"insertId": "x3kadao2vi2y3xsr",
"resource": {
"type": "k8s_container",
"labels": {
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth",
"container_name": "controller",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:02.218508766Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/type": "nginx",
"k8s-pod/stage": "main",
"k8s-pod/component": "controller",
"k8s-pod/app": "nginx-ingress",
"k8s-pod/shard": "default",
"k8s-pod/release": "gitlab",
"k8s-pod/deployment": "gitlab-nginx"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.161949943Z"
},
{
"textPayload": "34.65.5.0 - - [05/Apr/2024:19:19:02 +0000] \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/1.1\" 200 1173 \"-\" \"git/2.43.2\" 1577 0.127 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 1173 0.127 200 d72ba6529c1218c25aa94e094024e570",
"insertId": "tyby1taalnqu27re",
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth",
"cluster_name": "staging-ref-3k-hybrid-us",
"namespace_name": "default",
"location": "us-east1-c",
"project_id": "gitlab-staging-ref",
"container_name": "controller"
}
},
"timestamp": "2024-04-05T19:19:02.241278415Z",
"severity": "INFO",
"labels": {
"k8s-pod/component": "controller",
"k8s-pod/type": "nginx",
"k8s-pod/pod-template-hash": "6988b54ffb",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/shard": "default",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/release": "gitlab",
"k8s-pod/app": "nginx-ingress",
"k8s-pod/stage": "main"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.161949943Z"
},
{
"textPayload": "34.65.234.178 - - [05/Apr/2024:19:19:02 +0000] \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/1.1\" 200 4244 \"-\" \"git/2.43.2\" 1577 0.161 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 4249 0.161 200 9e1eff35afee24de0e58a13d25eab0da",
"insertId": "xpitsuuuh908abjc",
"resource": {
"type": "k8s_container",
"labels": {
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "controller",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-cvvjr",
"namespace_name": "default",
"location": "us-east1-c",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:02.262807734Z",
"severity": "INFO",
"labels": {
"k8s-pod/type": "nginx",
"k8s-pod/stage": "main",
"k8s-pod/shard": "default",
"k8s-pod/release": "gitlab",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/component": "controller",
"k8s-pod/app": "nginx-ingress",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/deployment": "gitlab-nginx"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.159041917Z"
},
{
"insertId": "kqq4mu4ntcgy4gcl",
"jsonPayload": {
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_count": 0,
"redis_repository_cache_write_bytes": 55,
"cpu_s": 0.046425,
"mem_mallocs": 2864,
"redis_write_bytes": 55,
"meta.root_namespace": "mkozonotest",
"db_ci_replica_txn_max_duration_s": 0,
"db_primary_txn_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"worker_id": "puma_2",
"db_ci_replica_wal_count": 0,
"redis_read_bytes": 101,
"meta.caller_id": "Repositories::GitHttpController#info_refs",
"db_main_txn_duration_s": 0,
"db_ci_txn_max_duration_s": 0,
"db_replica_txn_duration_s": 0,
"redis_duration_s": 0.000683,
"db_main_replica_txn_duration_s": 0,
"db_ci_replica_count": 0,
"db_primary_duration_s": 0,
"method": "GET",
"db_ci_replica_txn_duration_s": 0,
"meta.feature_category": "source_code_management",
"view_duration_s": 0.00026,
"repository_storage": "default",
"db_main_wal_count": 0,
"db_cached_count": 1,
"db_main_duration_s": 0,
"db_main_wal_cached_count": 0,
"mem_objects": 12623,
"db_main_replica_count": 10,
"db_txn_count": 0,
"db_ci_replica_duration_s": 0,
"db_count": 10,
"ua": "git/2.43.2",
"correlation_id": "01HTQV6EYA1MNZQ7DJ3CZ66M7V",
"request_urgency": "default",
"db_main_replica_txn_max_duration_s": 0,
"db_ci_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_primary_count": 0,
"db_replica_txn_max_duration_s": 0,
"status": 200,
"db_write_count": 0,
"format": "*/*",
"db_ci_wal_count": 0,
"db_main_replica_txn_count": 0,
"target_duration_s": 1,
"db_main_cached_count": 0,
"duration_s": 0.04344,
"meta.client_id": "ip/34.65.5.0",
"db_primary_wal_count": 0,
"path": "/mkozonotest/gitlab-runner.git/info/refs",
"db_ci_duration_s": 0,
"pid": 529,
"mem_bytes": 808368,
"controller": "Repositories::GitHttpController",
"db_main_replica_wal_cached_count": 0,
"component": "gitlab",
"db_main_txn_max_duration_s": 0,
"db_main_replica_duration_s": 0.008,
"db_replica_txn_count": 0,
"redis_calls": 1,
"redis_repository_cache_read_bytes": 101,
"remote_ip": "34.65.5.0",
"db_primary_txn_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_cached_count": 0,
"subcomponent": "production_json",
"params": [
{
"key": "service",
"value": "git-upload-pack"
},
{
"value": "mkozonotest/gitlab-runner.git",
"key": "repository_path"
}
],
"rate_limiting_gates": [],
"db_primary_cached_count": 0,
"redis_repository_cache_calls": 1,
"meta.project": "mkozonotest/gitlab-runner",
"db_main_replica_cached_count": 1,
"db_replica_wal_cached_count": 0,
"db_ci_txn_duration_s": 0,
"db_main_txn_count": 0,
"meta.remote_ip": "34.65.5.0",
"db_ci_replica_txn_count": 0,
"redis_repository_cache_duration_s": 0.000683,
"action": "info_refs",
"db_ci_replica_wal_cached_count": 0,
"db_ci_txn_count": 0,
"db_main_count": 0,
"db_replica_count": 10,
"db_replica_duration_s": 0.008,
"db_replica_cached_count": 1,
"db_ci_count": 0,
"mem_total_bytes": 1313288,
"db_duration_s": 0.00802
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"pod_name": "gitlab-webservice-default-669cf4599-qww8d",
"project_id": "gitlab-staging-ref",
"container_name": "webservice"
}
},
"timestamp": "2024-04-05T19:19:02.914Z",
"severity": "ERROR",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/tier": "sv",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/stage": "main",
"k8s-pod/type": "web",
"k8s-pod/app": "webservice",
"k8s-pod/shard": "default",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/heritage": "Helm"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:04.927265209Z"
},
{
"textPayload": "34.65.5.0 - - [05/Apr/2024:19:19:02 +0000] \"GET /mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack HTTP/1.1\" 200 160 \"-\" \"git/2.43.2\" 547 0.072 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 160 0.072 200 0f4ff392c7b16ee53bc55d811b208d34",
"insertId": "pi27qu5upybfjugd",
"resource": {
"type": "k8s_container",
"labels": {
"container_name": "controller",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth",
"location": "us-east1-c",
"project_id": "gitlab-staging-ref",
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-us"
}
},
"timestamp": "2024-04-05T19:19:02.930183355Z",
"severity": "INFO",
"labels": {
"k8s-pod/component": "controller",
"k8s-pod/stage": "main",
"k8s-pod/shard": "default",
"k8s-pod/release": "gitlab",
"k8s-pod/type": "nginx",
"k8s-pod/app": "nginx-ingress",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/pod-template-hash": "6988b54ffb"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.161949943Z"
},
{
"insertId": "a2dwaxna93g5obbt",
"jsonPayload": {
"referrer": "",
"remote_ip": "34.65.5.0",
"host": "staging-ref.gitlab.com",
"uri": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"msg": "access",
"level": "info",
"duration_ms": 79,
"correlation_id": "01HTQV6F3SK6Y9CDBM45X1QEEV",
"remote_addr": "34.65.5.0:58606",
"content_type": "application/x-git-upload-pack-result",
"system": "http",
"method": "POST",
"ttfb_ms": 77,
"route": "^/.+\\.git/git-upload-pack\\z",
"user_agent": "git/2.43.2",
"status": 200,
"written_bytes": 113705,
"proto": "HTTP/1.1"
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"cluster_name": "staging-ref-3k-hybrid-us",
"pod_name": "gitlab-webservice-default-669cf4599-qww8d",
"location": "us-east1-c",
"container_name": "gitlab-workhorse",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:03Z",
"severity": "INFO",
"labels": {
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/release": "gitlab",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/stage": "main",
"k8s-pod/app": "webservice",
"k8s-pod/heritage": "Helm",
"k8s-pod/shard": "default",
"k8s-pod/type": "web",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/tier": "sv"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.930552910Z"
},
{
"insertId": "90tb4xf47xnndwb9",
"jsonPayload": {
"db_count": 12,
"db_cached_count": 1,
"db_main_replica_txn_duration_s": 0,
"db_replica_txn_count": 0,
"target_duration_s": 1,
"meta.root_namespace": "mkozonotest",
"db_ci_replica_txn_max_duration_s": 0,
"db_main_replica_wal_cached_count": 0,
"correlation_id": "01HTQV6F3SK6Y9CDBM45X1QEEV",
"controller": "Repositories::GitHttpController",
"worker_id": "puma_2",
"db_primary_count": 1,
"db_main_txn_max_duration_s": 0,
"mem_objects": 13658,
"db_ci_txn_count": 0,
"rate_limiting_gates": [],
"db_main_cached_count": 0,
"meta.caller_id": "Repositories::GitHttpController#git_upload_pack",
"db_primary_txn_duration_s": 0,
"db_primary_cached_count": 0,
"db_ci_replica_txn_duration_s": 0,
"component": "gitlab",
"db_replica_wal_cached_count": 0,
"db_main_replica_wal_count": 0,
"db_replica_txn_max_duration_s": 0,
"status": 200,
"path": "/mkozonotest/gitlab-runner.git/git-upload-pack",
"redis_read_bytes": 101,
"redis_repository_cache_duration_s": 0.000708,
"cpu_s": 0.049384,
"db_main_replica_count": 11,
"db_main_wal_count": 0,
"mem_mallocs": 3019,
"mem_total_bytes": 1367264,
"db_ci_replica_duration_s": 0,
"db_ci_count": 0,
"db_write_count": 1,
"db_main_replica_cached_count": 1,
"format": null,
"db_ci_txn_duration_s": 0,
"redis_repository_cache_read_bytes": 101,
"redis_duration_s": 0.000708,
"request_urgency": "default",
"db_ci_wal_count": 0,
"db_ci_replica_txn_count": 0,
"db_primary_txn_count": 0,
"db_primary_wal_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_replica_cached_count": 1,
"view_duration_s": 0.00026,
"db_ci_wal_cached_count": 0,
"action": "git_upload_pack",
"db_replica_wal_count": 0,
"remote_ip": "34.65.5.0",
"redis_repository_cache_calls": 1,
"subcomponent": "production_json",
"db_primary_txn_max_duration_s": 0,
"db_replica_count": 11,
"db_ci_txn_max_duration_s": 0,
"mem_bytes": 820944,
"db_duration_s": 0.01275,
"redis_calls": 1,
"db_main_duration_s": 0.003,
"method": "POST",
"db_txn_count": 0,
"db_main_txn_count": 0,
"db_ci_cached_count": 0,
"db_ci_replica_wal_count": 0,
"redis_repository_cache_write_bytes": 55,
"db_replica_txn_duration_s": 0,
"db_ci_duration_s": 0,
"meta.feature_category": "source_code_management",
"db_main_replica_txn_max_duration_s": 0,
"db_main_replica_txn_count": 0,
"redis_write_bytes": 55,
"db_ci_replica_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"repository_storage": "default",
"db_replica_duration_s": 0.01,
"duration_s": 0.05009,
"db_ci_replica_count": 0,
"params": [
{
"value": "mkozonotest/gitlab-runner.git",
"key": "repository_path"
}
],
"meta.project": "mkozonotest/gitlab-runner",
"db_primary_wal_count": 0,
"meta.remote_ip": "34.65.5.0",
"meta.client_id": "ip/34.65.5.0",
"db_main_replica_duration_s": 0.01,
"db_main_txn_duration_s": 0,
"db_primary_duration_s": 0.003,
"db_main_count": 1,
"pid": 529,
"ua": "git/2.43.2"
},
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-webservice-default-669cf4599-qww8d",
"project_id": "gitlab-staging-ref",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "webservice",
"namespace_name": "default",
"location": "us-east1-c"
}
},
"timestamp": "2024-04-05T19:19:03.097Z",
"severity": "ERROR",
"labels": {
"k8s-pod/type": "web",
"k8s-pod/shard": "default",
"k8s-pod/release": "gitlab",
"k8s-pod/app": "webservice",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/heritage": "Helm",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-8j2b",
"k8s-pod/tier": "sv",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/stage": "main"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:04.927265209Z"
},
{
"textPayload": "34.65.5.0 - - [05/Apr/2024:19:19:03 +0000] \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/1.1\" 200 113933 \"-\" \"git/2.43.2\" 721 0.181 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 113750 0.181 200 727127bf75ab2e5ad745aea605ba14ce",
"insertId": "o4hvarvvs9fsybrv",
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth",
"location": "us-east1-c",
"namespace_name": "default",
"project_id": "gitlab-staging-ref",
"container_name": "controller",
"cluster_name": "staging-ref-3k-hybrid-us"
}
},
"timestamp": "2024-04-05T19:19:03.215206722Z",
"severity": "INFO",
"labels": {
"k8s-pod/deployment": "gitlab-nginx",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/shard": "default",
"k8s-pod/app": "nginx-ingress",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/release": "gitlab",
"k8s-pod/stage": "main",
"k8s-pod/component": "controller",
"k8s-pod/type": "nginx"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:04.161949943Z"
},
{
"insertId": "24q2mvdzg9tgv7lh",
"jsonPayload": {
"uri": "/mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack",
"status": 200,
"remote_ip": "34.65.54.135",
"proto": "HTTP/1.1",
"ttfb_ms": 277,
"duration_ms": 287,
"written_bytes": 51544,
"level": "info",
"host": "staging-ref.gitlab.com",
"referrer": "",
"method": "GET",
"system": "http",
"user_agent": "GitLab-Shell",
"content_type": "application/x-git-upload-pack-advertisement",
"remote_addr": "34.65.54.135:42450",
"correlation_id": "01HTQV6J7J55GYN0GVGWWZ1GDT",
"msg": "access",
"route": "^/.+\\.git/info/refs\\z"
},
"resource": {
"type": "k8s_container",
"labels": {
"cluster_name": "staging-ref-3k-hybrid-us",
"location": "us-east1-c",
"container_name": "gitlab-workhorse",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"project_id": "gitlab-staging-ref",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:06Z",
"severity": "INFO",
"labels": {
"k8s-pod/release": "gitlab",
"k8s-pod/app": "webservice",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/gitlab_com/webservice-name": "default",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/heritage": "Helm",
"k8s-pod/stage": "main",
"k8s-pod/tier": "sv",
"k8s-pod/type": "web",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/shard": "default"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:10.891093040Z"
},
{
"insertId": "2mncz82wq8s35z3w",
"jsonPayload": {
"redis_feature_flag_duration_s": 0.00088,
"db_primary_duration_s": 0,
"db_ci_replica_cached_count": 0,
"format": "html",
"db_main_replica_duration_s": 0.196,
"db_ci_replica_wal_cached_count": 0,
"db_ci_replica_duration_s": 0,
"params": [
{
"key": "service",
"value": "git-upload-pack"
},
{
"key": "repository_path",
"value": "mkozonotest/gitlab-runner.git"
}
],
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_replica_txn_count": 0,
"db_cached_count": 2,
"db_main_count": 0,
"target_duration_s": 1,
"method": "GET",
"db_duration_s": 0.19906,
"request_urgency": "default",
"controller": "Repositories::GitHttpController",
"meta.root_namespace": "mkozonotest",
"redis_repository_cache_calls": 1,
"db_replica_duration_s": 0.196,
"db_ci_replica_txn_max_duration_s": 0,
"meta.project": "mkozonotest/gitlab-runner",
"db_ci_txn_max_duration_s": 0,
"redis_repository_cache_write_bytes": 55,
"db_replica_count": 17,
"cpu_s": 0.069342,
"db_replica_wal_count": 0,
"db_main_cached_count": 0,
"redis_repository_cache_read_bytes": 101,
"db_main_replica_txn_max_duration_s": 0,
"repository_storage": "default",
"rate_limiting_gates": [],
"action": "info_refs",
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_ci_replica_txn_count": 0,
"redis_feature_flag_calls": 2,
"ua": "GitLab-Shell",
"redis_calls": 3,
"redis_repository_cache_duration_s": 0.000518,
"worker_id": "puma_3",
"db_ci_wal_cached_count": 0,
"db_write_count": 0,
"redis_feature_flag_read_bytes": 2,
"redis_feature_flag_write_bytes": 376,
"db_replica_txn_max_duration_s": 0,
"db_ci_duration_s": 0,
"db_main_txn_count": 0,
"mem_total_bytes": 4606104,
"correlation_id": "01HTQV6J7J55GYN0GVGWWZ1GDT",
"redis_duration_s": 0.001398,
"db_main_replica_wal_cached_count": 0,
"redis_read_bytes": 103,
"db_primary_count": 0,
"db_main_replica_count": 17,
"db_ci_replica_count": 0,
"meta.remote_ip": "34.65.54.135",
"db_ci_cached_count": 0,
"status": 200,
"db_main_duration_s": 0,
"db_main_wal_cached_count": 0,
"remote_ip": "34.65.54.135",
"meta.feature_category": "source_code_management",
"db_ci_replica_wal_count": 0,
"db_main_wal_count": 0,
"view_duration_s": 0.00021,
"db_ci_replica_txn_duration_s": 0,
"db_main_replica_txn_count": 0,
"subcomponent": "production_json",
"db_main_replica_txn_duration_s": 0,
"db_replica_txn_duration_s": 0,
"db_primary_cached_count": 0,
"mem_bytes": 3603384,
"db_ci_count": 0,
"db_primary_wal_count": 0,
"meta.caller_id": "Repositories::GitHttpController#info_refs",
"path": "/mkozonotest/gitlab-runner.git/info/refs",
"db_primary_txn_duration_s": 0,
"db_txn_count": 0,
"component": "gitlab",
"db_replica_cached_count": 2,
"db_ci_txn_count": 0,
"db_ci_txn_duration_s": 0,
"mem_mallocs": 6950,
"db_main_txn_max_duration_s": 0,
"db_count": 17,
"db_main_replica_cached_count": 2,
"db_primary_txn_count": 0,
"duration_s": 0.25844,
"pid": 380,
"meta.client_id": "ip/34.65.54.135",
"redis_write_bytes": 431,
"db_main_txn_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"mem_objects": 25068
},
"resource": {
"type": "k8s_container",
"labels": {
"namespace_name": "default",
"pod_name": "gitlab-webservice-default-669cf4599-6jjpx",
"cluster_name": "staging-ref-3k-hybrid-us",
"container_name": "webservice",
"location": "us-east1-c",
"project_id": "gitlab-staging-ref"
}
},
"timestamp": "2024-04-05T19:19:06.496Z",
"severity": "ERROR",
"labels": {
"k8s-pod/heritage": "Helm",
"k8s-pod/chart": "webservice-7.10.1",
"k8s-pod/type": "web",
"k8s-pod/app": "webservice",
"k8s-pod/release": "gitlab",
"k8s-pod/gitlab_com/webservice-name": "default",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-webservice-20-7039a04c-lvpq",
"k8s-pod/stage": "main",
"k8s-pod/tier": "sv",
"k8s-pod/pod-template-hash": "669cf4599",
"k8s-pod/shard": "default"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:10.891233720Z"
},
{
"textPayload": "34.65.54.135 - - [05/Apr/2024:19:19:06 +0000] \"GET /mkozonotest/gitlab-runner.git/info/refs?service=git-upload-pack HTTP/2.0\" 200 51544 \"-\" \"GitLab-Shell\" 392 0.289 [default-gitlab-webservice-default-8181] [] 10.44.12.146:8181 51731 0.289 200 01HTQV6DCA5NEH56B7QKKF00PM",
"insertId": "u4l1wcf8idx1tluu",
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-7ssth",
"container_name": "controller",
"cluster_name": "staging-ref-3k-hybrid-us",
"location": "us-east1-c",
"namespace_name": "default"
}
},
"timestamp": "2024-04-05T19:19:06.514562160Z",
"severity": "INFO",
"labels": {
"k8s-pod/stage": "main",
"k8s-pod/type": "nginx",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/release": "gitlab",
"k8s-pod/shard": "default",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/component": "controller",
"k8s-pod/app": "nginx-ingress",
"k8s-pod/deployment": "gitlab-nginx"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:19:09.277886157Z"
},
{
"textPayload": "2024/04/05 19:19:07 [warn] 241#241: *21788603 a client request body is buffered to a temporary file /tmp/nginx/client-body/0000002691, client: 34.65.54.135, server: staging-ref.gitlab.com, request: \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/2.0\", host: \"staging-ref.gitlab.com\"",
"insertId": "lg6ny3k506vowg6p",
"resource": {
"type": "k8s_container",
"labels": {
"project_id": "gitlab-staging-ref",
"location": "us-east1-c",
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-cvvjr",
"cluster_name": "staging-ref-3k-hybrid-us",
"namespace_name": "default",
"container_name": "controller"
}
},
"timestamp": "2024-04-05T19:19:07.736116968Z",
"severity": "ERROR",
"labels": {
"k8s-pod/app": "nginx-ingress",
"k8s-pod/type": "nginx",
"k8s-pod/stage": "main",
"k8s-pod/component": "controller",
"k8s-pod/release": "gitlab",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/deployment": "gitlab-nginx",
"k8s-pod/shard": "default"
},
"logName": "projects/gitlab-staging-ref/logs/stderr",
"receiveTimestamp": "2024-04-05T19:19:09.322251515Z"
},
{
"textPayload": "34.65.54.135 - - [05/Apr/2024:19:20:07 +0000] \"POST /mkozonotest/gitlab-runner.git/git-upload-pack HTTP/2.0\" 408 0 \"-\" \"GitLab-Shell\" 2203 60.689 [default-gitlab-webservice-default-8181] [] - - - - 01HTQV6DCA5NEH56B7QKKF00PM",
"insertId": "dqhdeai5e3lfm8ab",
"resource": {
"type": "k8s_container",
"labels": {
"pod_name": "gitlab-nginx-ingress-controller-6988b54ffb-cvvjr",
"location": "us-east1-c",
"cluster_name": "staging-ref-3k-hybrid-us",
"project_id": "gitlab-staging-ref",
"namespace_name": "default",
"container_name": "controller"
}
},
"timestamp": "2024-04-05T19:20:07.736114441Z",
"severity": "INFO",
"labels": {
"k8s-pod/app": "nginx-ingress",
"k8s-pod/component": "controller",
"compute.googleapis.com/resource_name": "gke-staging-ref-3k-h-gl-supporting-20-b6153d9e-8qc9",
"k8s-pod/stage": "main",
"k8s-pod/release": "gitlab",
"k8s-pod/shard": "default",
"k8s-pod/type": "nginx",
"k8s-pod/pod-template-hash": "6988b54ffb",
"k8s-pod/deployment": "gitlab-nginx"
},
"logName": "projects/gitlab-staging-ref/logs/stdout",
"receiveTimestamp": "2024-04-05T19:20:09.164206686Z"
}
]Workarounds
Users could attempt to fetch again, and hope that the secondary is up-to-date and therefore not proxy. This workaround is less effective on large, highly-active repositories.
Edited Apr 05, 2024 by Michael Kozono