From fb5000e83ded8f629eddc06d49f8d1a270a85c65 Mon Sep 17 00:00:00 2001 From: ex5 <176934+ex5@users.noreply.github.com> Date: Tue, 10 Oct 2023 19:16:34 +0200 Subject: [PATCH 1/4] docs(vrl): add an example of parsing upstreaminfo with parse_nginx_log --- .../remap/functions/parse_nginx_log.cue | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/website/cue/reference/remap/functions/parse_nginx_log.cue b/website/cue/reference/remap/functions/parse_nginx_log.cue index 1234dc99284a1..95a6f379dce40 100644 --- a/website/cue/reference/remap/functions/parse_nginx_log.cue +++ b/website/cue/reference/remap/functions/parse_nginx_log.cue @@ -93,5 +93,31 @@ remap: functions: parse_nginx_log: { host: "localhost:8081" } }, + { + title: "Parse via Nginx log format (ingress_upstreaminfo)" + source: #""" + parse_nginx_log!( + s'0.0.0.0 - - [18/Mar/2023:15:00:00 +0000] "GET /some/path HTTP/2.0" 200 12312 "https://10.0.0.1/some/referer" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" 462 0.050 [some-upstream-service-9000] [] 10.0.0.2:9000 12123 0.049 200 752178deadbeef386279e7', + "ingress_upstreaminfo" + ) + """# + return: { + body_bytes_size: 12312 + http_referer: "https://10.0.0.1/some/referer" + http_user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" + proxy_upstream_name: "some-upstream-service-9000" + remote_addr: "0.0.0.0" + req_id: "752178deadbeef386279e7" + request: "GET /some/path HTTP/2.0" + request_length: 462 + request_time: 0.050 + status: 200 + timestamp: "2023-03-18T15:00:00Z" + upstream_addr: "10.0.0.2:9000" + upstream_response_length: 12123 + upstream_response_time: 0.049 + upstream_status: 200 + } + }, ] } From ff35bb7cb642aa6c73f0d47a39317db2e5e01f71 Mon Sep 17 00:00:00 2001 From: ex5 <176934+ex5@users.noreply.github.com> Date: Tue, 10 Oct 2023 19:25:26 +0200 Subject: [PATCH 2/4] Copy the example message from the "all valid" test --- .../remap/functions/parse_nginx_log.cue | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/website/cue/reference/remap/functions/parse_nginx_log.cue b/website/cue/reference/remap/functions/parse_nginx_log.cue index 95a6f379dce40..0e5d2b068286b 100644 --- a/website/cue/reference/remap/functions/parse_nginx_log.cue +++ b/website/cue/reference/remap/functions/parse_nginx_log.cue @@ -97,26 +97,28 @@ remap: functions: parse_nginx_log: { title: "Parse via Nginx log format (ingress_upstreaminfo)" source: #""" parse_nginx_log!( - s'0.0.0.0 - - [18/Mar/2023:15:00:00 +0000] "GET /some/path HTTP/2.0" 200 12312 "https://10.0.0.1/some/referer" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" 462 0.050 [some-upstream-service-9000] [] 10.0.0.2:9000 12123 0.049 200 752178deadbeef386279e7', + s'0.0.0.0 - bob [18/Mar/2023:15:00:00 +0000] "GET /some/path HTTP/2.0" 200 12312 "https://10.0.0.1/some/referer" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" 462 0.050 [some-upstream-service-9000] [some-other-upstream-5000] 10.0.50.80:9000 19437 0.049 200 752178adb17130b291aefd8c386279e7', "ingress_upstreaminfo" ) """# return: { - body_bytes_size: 12312 - http_referer: "https://10.0.0.1/some/referer" - http_user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" - proxy_upstream_name: "some-upstream-service-9000" - remote_addr: "0.0.0.0" - req_id: "752178deadbeef386279e7" - request: "GET /some/path HTTP/2.0" - request_length: 462 - request_time: 0.050 - status: 200 - timestamp: "2023-03-18T15:00:00Z" - upstream_addr: "10.0.0.2:9000" - upstream_response_length: 12123 - upstream_response_time: 0.049 - upstream_status: 200 + body_bytes_size: 12312 + http_referer: "https://10.0.0.1/some/referer" + http_user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" + proxy_alternative_upstream_name: "some-other-upstream-5000" + proxy_upstream_name: "some-upstream-service-9000" + remote_addr: "0.0.0.0" + remote_user: "bob" + req_id: "752178adb17130b291aefd8c386279e7" + request: "GET /some/path HTTP/2.0" + request_length: 462 + request_time: 0.050 + status: 200 + timestamp: "2023-03-18T15:00:00Z" + upstream_addr: "10.0.50.80:9000" + upstream_response_length: 19437 + upstream_response_time: 0.049 + upstream_status: 200 } }, ] From 5c50174b5475855913345b2b82d1afab81f41e41 Mon Sep 17 00:00:00 2001 From: ex5 <176934+ex5@users.noreply.github.com> Date: Wed, 11 Oct 2023 18:48:43 +0200 Subject: [PATCH 3/4] Mention which fields might not be present when parsing a ingress_upstreaminfo log --- website/cue/reference/remap/functions/parse_nginx_log.cue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/cue/reference/remap/functions/parse_nginx_log.cue b/website/cue/reference/remap/functions/parse_nginx_log.cue index 0e5d2b068286b..844ca6de37b05 100644 --- a/website/cue/reference/remap/functions/parse_nginx_log.cue +++ b/website/cue/reference/remap/functions/parse_nginx_log.cue @@ -10,6 +10,9 @@ remap: functions: parse_nginx_log: { """ Missing information in the log message may be indicated by `-`. These fields are omitted in the result. """, + """ + In case of `ingress_upstreaminfo` format the following fields may be safely omitted in the log message: `remote_addr`, `remote_user`, `http_referer`, `http_user_agent`, `proxy_alternative_upstream_name`, `upstream_addr`, `upstream_response_length`, `upstream_response_time`, `upstream_status`. + """, ] arguments: [ From 8bc85614073e42e51f731a4bd3de85249384f72c Mon Sep 17 00:00:00 2001 From: ex5 <176934+ex5@users.noreply.github.com> Date: Wed, 11 Oct 2023 18:49:20 +0200 Subject: [PATCH 4/4] Replace whitespaces breaking CUE validation --- website/cue/reference/remap/functions/parse_nginx_log.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/cue/reference/remap/functions/parse_nginx_log.cue b/website/cue/reference/remap/functions/parse_nginx_log.cue index 844ca6de37b05..dd4078aca8588 100644 --- a/website/cue/reference/remap/functions/parse_nginx_log.cue +++ b/website/cue/reference/remap/functions/parse_nginx_log.cue @@ -100,7 +100,7 @@ remap: functions: parse_nginx_log: { title: "Parse via Nginx log format (ingress_upstreaminfo)" source: #""" parse_nginx_log!( - s'0.0.0.0 - bob [18/Mar/2023:15:00:00 +0000] "GET /some/path HTTP/2.0" 200 12312 "https://10.0.0.1/some/referer" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" 462 0.050 [some-upstream-service-9000] [some-other-upstream-5000] 10.0.50.80:9000 19437 0.049 200 752178adb17130b291aefd8c386279e7', + s'0.0.0.0 - bob [18/Mar/2023:15:00:00 +0000] "GET /some/path HTTP/2.0" 200 12312 "https://10.0.0.1/some/referer" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" 462 0.050 [some-upstream-service-9000] [some-other-upstream-5000] 10.0.50.80:9000 19437 0.049 200 752178adb17130b291aefd8c386279e7', "ingress_upstreaminfo" ) """#