From ea2d57667ebe5bbea2124403b94febc702fcf759 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 14:41:31 -0400 Subject: [PATCH 01/12] chore(deps): Bump serde from 1.0.183 to 1.0.185 (#18319) * chore(deps): Bump serde from 1.0.183 to 1.0.185 Bumps [serde](https://github.com/serde-rs/serde) from 1.0.183 to 1.0.185. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.183...v1.0.185) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * update LICENSE-3rdparty.csv --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Nathan Fox --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 - lib/vector-api-client/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 2 +- lib/vector-config-macros/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- lib/vector-lookup/Cargo.toml | 2 +- 9 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0a3a16bafa26..82c5c98135468 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7557,9 +7557,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.183" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" dependencies = [ "serde_derive", ] @@ -7605,9 +7605,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", diff --git a/Cargo.toml b/Cargo.toml index 4cfa9f547b5b8..bc14965f2dcc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -201,7 +201,7 @@ opendal = {version = "0.38", default-features = false, features = ["native-tls", tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } tower-http = { version = "0.4.3", default-features = false, features = ["decompression-gzip"]} # Serde -serde = { version = "1.0.183", default-features = false, features = ["derive"] } +serde = { version = "1.0.185", default-features = false, features = ["derive"] } serde-toml-merge = { version = "0.3.0", default-features = false } serde_bytes = { version = "0.11.12", default-features = false, features = ["std"], optional = true } serde_json = { version = "1.0.105", default-features = false, features = ["raw_value"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 3f4f81b68a599..643187608c34d 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -483,7 +483,6 @@ serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx serde_bytes,https://github.com/serde-rs/bytes,MIT OR Apache-2.0,David Tolnay -serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,David Tolnay serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_nanos,https://github.com/caspervonb/serde_nanos,MIT OR Apache-2.0,Casper Beyer serde_path_to_error,https://github.com/dtolnay/path-to-error,MIT OR Apache-2.0,David Tolnay diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 7e544cfe11578..ff42a6d32c62d 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -9,7 +9,7 @@ license = "MPL-2.0" [dependencies] # Serde -serde = { version = "1.0.183", default-features = false, features = ["derive"] } +serde = { version = "1.0.185", default-features = false, features = ["derive"] } serde_json = { version = "1.0.105", default-features = false, features = ["raw_value"] } # Error handling diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index e68712058c189..aa37e0d02b769 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -21,7 +21,7 @@ metrics = "0.21.1" num-traits = { version = "0.2.16", default-features = false } pin-project.workspace = true rkyv = { version = "0.7.40", default-features = false, features = ["size_32", "std", "strict", "validation"] } -serde = { version = "1.0.183", default-features = false, features = ["derive"] } +serde = { version = "1.0.185", default-features = false, features = ["derive"] } snafu = { version = "0.7.5", default-features = false, features = ["std"] } tokio-util = { version = "0.7.0", default-features = false } tokio = { version = "1.32.0", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 1d55460205a5a..3ff2e7a308bd8 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -56,7 +56,7 @@ paste = "1.0.14" pin-project.workspace = true ryu = { version = "1", default-features = false } serde_json = { version = "1.0.105", default-features = false, features = ["std", "raw_value"] } -serde = { version = "1.0.183", optional = true, features = ["derive"] } +serde = { version = "1.0.185", optional = true, features = ["derive"] } smallvec = { version = "1", default-features = false } snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } diff --git a/lib/vector-config-macros/Cargo.toml b/lib/vector-config-macros/Cargo.toml index 60c0e9b3523d6..1caeb8f07aa82 100644 --- a/lib/vector-config-macros/Cargo.toml +++ b/lib/vector-config-macros/Cargo.toml @@ -16,5 +16,5 @@ syn = { version = "2.0", default-features = false, features = ["full", "extra-tr vector-config-common = { path = "../vector-config-common" } [dev-dependencies] -serde = { version = "1.0.183", default-features = false } +serde = { version = "1.0.185", default-features = false } vector-config = { path = "../vector-config" } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 91b037b4b629b..65b5998930c9b 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -41,7 +41,7 @@ prost = { version = "0.11", default-features = false, features = ["std"] } quanta = { version = "0.11.1", default-features = false } regex = { version = "1.9.3", default-features = false, features = ["std", "perf"] } ryu = { version = "1", default-features = false } -serde = { version = "1.0.183", default-features = false, features = ["derive", "rc"] } +serde = { version = "1.0.185", default-features = false, features = ["derive", "rc"] } serde_json = { version = "1.0.105", default-features = false } serde_with = { version = "3.2.0", default-features = false, features = ["std", "macros"] } smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } diff --git a/lib/vector-lookup/Cargo.toml b/lib/vector-lookup/Cargo.toml index 9d4d0db3965a2..bddfafc49e62f 100644 --- a/lib/vector-lookup/Cargo.toml +++ b/lib/vector-lookup/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MPL-2.0" [dependencies] -serde = { version = "1.0.183", default-features = false, features = ["derive", "alloc"] } +serde = { version = "1.0.185", default-features = false, features = ["derive", "alloc"] } vector-config = { path = "../vector-config" } vector-config-macros = { path = "../vector-config-macros" } vrl.workspace = true From b3a1d5307453e5c4ba70c3b2bde45f879f5b9973 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 18:41:44 +0000 Subject: [PATCH 02/12] chore(deps): Bump similar-asserts from 1.4.2 to 1.5.0 (#18318) Bumps [similar-asserts](https://github.com/mitsuhiko/similar-asserts) from 1.4.2 to 1.5.0. - [Changelog](https://github.com/mitsuhiko/similar-asserts/blob/main/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/similar-asserts/compare/1.4.2...1.5.0) --- updated-dependencies: - dependency-name: similar-asserts dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/codecs/Cargo.toml | 2 +- lib/file-source/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82c5c98135468..c914f567455ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7923,9 +7923,9 @@ dependencies = [ [[package]] name = "similar-asserts" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf644ad016b75129f01a34a355dcb8d66a5bc803e417c7a77cc5d5ee9fa0f18" +checksum = "e041bb827d1bfca18f213411d51b665309f1afb37a04a5d1464530e13779fc0f" dependencies = [ "console", "similar", diff --git a/Cargo.toml b/Cargo.toml index bc14965f2dcc3..d0300a25f2cb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -362,7 +362,7 @@ base64 = "0.21.2" criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } itertools = { version = "0.11.0", default-features = false, features = ["use_alloc"] } libc = "0.2.147" -similar-asserts = "1.4.2" +similar-asserts = "1.5.0" proptest = "1.2" quickcheck = "1.0.3" lookup = { package = "vector-lookup", path = "lib/vector-lookup", features = ["test"] } diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index e46004bfc551e..25b26092ee45d 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -37,7 +37,7 @@ vector-core = { path = "../vector-core", default-features = false } futures = { version = "0.3", default-features = false } indoc = { version = "2", default-features = false } tokio = { version = "1", features = ["test-util"] } -similar-asserts = "1.4.2" +similar-asserts = "1.5.0" [features] syslog = ["dep:syslog_loose"] diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index d346c994b163d..3415511f7bf9e 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -77,7 +77,7 @@ features = ["full"] criterion = "0.5" quickcheck = "1" tempfile = "3.6.0" -similar-asserts = "1.4.2" +similar-asserts = "1.5.0" [[bench]] name = "buffer" diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 65b5998930c9b..cda0cbbda83e8 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -85,7 +85,7 @@ env-test-util = "1.0.1" quickcheck = "1" quickcheck_macros = "1" proptest = "1.2" -similar-asserts = "1.4.2" +similar-asserts = "1.5.0" tokio-test = "0.4.2" toml = { version = "0.7.6", default-features = false, features = ["parse"] } ndarray = "0.15.6" From e9637665b04a0dfe2a785673607f7592ee8fecac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 18:42:21 +0000 Subject: [PATCH 03/12] chore(deps): Bump notify from 6.0.1 to 6.1.0 (#18317) * chore(deps): Bump notify from 6.0.1 to 6.1.0 Bumps [notify](https://github.com/notify-rs/notify) from 6.0.1 to 6.1.0. - [Release notes](https://github.com/notify-rs/notify/releases) - [Changelog](https://github.com/notify-rs/notify/blob/main/CHANGELOG.md) - [Commits](https://github.com/notify-rs/notify/compare/notify-6.0.1...notify-6.1.0) --- updated-dependencies: - dependency-name: notify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * update LICENSE-3rdparty.csv --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Nathan Fox --- Cargo.lock | 17 +++++++++-------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c914f567455ed..729caf188989c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3359,14 +3359,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.18" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b9663d381d07ae25dc88dbdf27df458faa83a9b25336bcac83d5e452b5fc9d3" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", - "windows-sys 0.42.0", + "redox_syscall 0.3.5", + "windows-sys 0.48.0", ] [[package]] @@ -5494,19 +5494,20 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" [[package]] name = "notify" -version = "6.0.1" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5738a2795d57ea20abec2d6d76c6081186709c0024187cd5977265eda6598b51" +checksum = "3dbf4f3b058c29e9642cf53217e0531cbfc78bc24e0a212a9837b7415b9d9007" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.3.2", "filetime", "fsevent-sys", "inotify", "kqueue", "libc", + "log", "mio", "walkdir", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d0300a25f2cb1..9d61788d2101f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -295,7 +295,7 @@ mongodb = { version = "2.6.1", default-features = false, features = ["tokio-runt async-nats = { version = "0.31.0", default-features = false, optional = true } nkeys = { version = "0.3.1", default-features = false, optional = true } nom = { version = "7.1.3", default-features = false, optional = true } -notify = { version = "6.0.1", default-features = false, features = ["macos_fsevent"] } +notify = { version = "6.1.0", default-features = false, features = ["macos_fsevent"] } once_cell = { version = "1.18", default-features = false } openssl = { version = "0.10.56", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.5", default-features = false } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 643187608c34d..a2f2fa99215cc 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -345,7 +345,7 @@ no-proxy,https://github.com/jdrouet/no-proxy,MIT,Jérémie Drouet nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com nonzero_ext,https://github.com/antifuchs/nonzero_ext,Apache-2.0,Andreas Fuchs -notify,https://github.com/notify-rs/notify,CC0-1.0 OR Artistic-2.0,"Félix Saparelli , Daniel Faust " +notify,https://github.com/notify-rs/notify,CC0-1.0,"Félix Saparelli , Daniel Faust , Aron Heinecke " ntapi,https://github.com/MSxDOS/ntapi,Apache-2.0 OR MIT,MSxDOS nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers" nuid,https://github.com/casualjim/rs-nuid,Apache-2.0,Ivan Porto Carrero From 54d48d79b5ed569a8e213c697a79dd0d91177d2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 19:06:01 +0000 Subject: [PATCH 04/12] chore(deps): Bump serde_with from 3.2.0 to 3.3.0 (#18315) Bumps [serde_with](https://github.com/jonasbb/serde_with) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/jonasbb/serde_with/releases) - [Commits](https://github.com/jonasbb/serde_with/compare/v3.2.0...v3.3.0) --- updated-dependencies: - dependency-name: serde_with dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- lib/vector-config/Cargo.toml | 4 ++-- lib/vector-core/Cargo.toml | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 729caf188989c..324b4934c069d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7726,9 +7726,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1402f54f9a3b9e2efe71c1cea24e648acce55887983553eeb858cf3115acfd49" +checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237" dependencies = [ "base64 0.21.2", "chrono", @@ -7737,7 +7737,7 @@ dependencies = [ "indexmap 2.0.0", "serde", "serde_json", - "serde_with_macros 3.2.0", + "serde_with_macros 3.3.0", "time", ] @@ -7755,9 +7755,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9197f1ad0e3c173a0222d3c4404fb04c3afe87e962bcb327af73e8301fa203c7" +checksum = "2e6be15c453eb305019bfa438b1593c731f36a289a7853f7707ee29e870b3b3c" dependencies = [ "darling 0.20.3", "proc-macro2 1.0.66", @@ -9553,7 +9553,7 @@ dependencies = [ "serde-toml-merge", "serde_bytes", "serde_json", - "serde_with 3.2.0", + "serde_with 3.3.0", "serde_yaml 0.9.25", "sha2", "similar-asserts", @@ -9721,7 +9721,7 @@ dependencies = [ "once_cell", "serde", "serde_json", - "serde_with 3.2.0", + "serde_with 3.3.0", "snafu", "toml 0.7.6", "tracing 0.1.37", @@ -9814,7 +9814,7 @@ dependencies = [ "security-framework", "serde", "serde_json", - "serde_with 3.2.0", + "serde_with 3.3.0", "similar-asserts", "smallvec", "snafu", diff --git a/Cargo.toml b/Cargo.toml index 9d61788d2101f..614a3ccb8748c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -205,7 +205,7 @@ serde = { version = "1.0.185", default-features = false, features = ["derive"] } serde-toml-merge = { version = "0.3.0", default-features = false } serde_bytes = { version = "0.11.12", default-features = false, features = ["std"], optional = true } serde_json = { version = "1.0.105", default-features = false, features = ["raw_value"] } -serde_with = { version = "3.2.0", default-features = false, features = ["macros", "std"] } +serde_with = { version = "3.3.0", default-features = false, features = ["macros", "std"] } serde_yaml = { version = "0.9.25", default-features = false } # Messagepack diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index d8ff34b2e41de..71f86425f7df6 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -21,7 +21,7 @@ num-traits = { version = "0.2.16", default-features = false } once_cell = { version = "1", default-features = false } serde = { version = "1.0", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["std"] } -serde_with = { version = "3.2.0", default-features = false, features = ["std"] } +serde_with = { version = "3.3.0", default-features = false, features = ["std"] } snafu = { version = "0.7.5", default-features = false } toml = { version = "0.7.6", default-features = false } tracing = { version = "0.1.34", default-features = false } @@ -33,4 +33,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] assert-json-diff = { version = "2", default-features = false } -serde_with = { version = "3.2.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.3.0", default-features = false, features = ["std", "macros"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index cda0cbbda83e8..efa202d54450e 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -43,7 +43,7 @@ regex = { version = "1.9.3", default-features = false, features = ["std", "perf" ryu = { version = "1", default-features = false } serde = { version = "1.0.185", default-features = false, features = ["derive", "rc"] } serde_json = { version = "1.0.105", default-features = false } -serde_with = { version = "3.2.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.3.0", default-features = false, features = ["std", "macros"] } smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu = { version = "0.7.5", default-features = false } socket2 = { version = "0.5.3", default-features = false } From a7800f74b0cdd75415b6852e37e03dbd4a4f4e27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 19:25:26 +0000 Subject: [PATCH 05/12] chore(deps): Bump tokio-postgres from 0.7.7 to 0.7.9 (#18316) * chore(deps): Bump tokio-postgres from 0.7.7 to 0.7.9 Bumps [tokio-postgres](https://github.com/sfackler/rust-postgres) from 0.7.7 to 0.7.9. - [Release notes](https://github.com/sfackler/rust-postgres/releases) - [Commits](https://github.com/sfackler/rust-postgres/compare/tokio-postgres-v0.7.7...tokio-postgres-v0.7.9) --- updated-dependencies: - dependency-name: tokio-postgres dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * update LICENSE-3rdparty.csv --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Nathan Fox --- Cargo.lock | 28 ++++++++++++++++++++-------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 + 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 324b4934c069d..d7aae976ecea4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6298,11 +6298,11 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" +checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.13.1", + "base64 0.21.2", "byteorder", "bytes 1.4.0", "fallible-iterator", @@ -6316,9 +6316,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73d946ec7d256b04dfadc4e6a3292324e6f417124750fc5c0950f981b703a0f1" +checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" dependencies = [ "bytes 1.4.0", "chrono", @@ -8576,9 +8576,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.7" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a12c1b3e0704ae7dfc25562629798b29c72e6b1d0a681b6f29ab4ae5e7f7bf" +checksum = "000387915083ea6406ee44b50ca74813aba799fe682a7689e382bf9e13b74ce9" dependencies = [ "async-trait", "byteorder", @@ -8593,9 +8593,11 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2 0.4.9", + "rand 0.8.5", + "socket2 0.5.3", "tokio", "tokio-util", + "whoami", ] [[package]] @@ -10251,6 +10253,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index 614a3ccb8748c..73be098427db7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -320,7 +320,7 @@ stream-cancel = { version = "0.8.1", default-features = false } strip-ansi-escapes = { version = "0.2.0", default-features = false } syslog = { version = "6.1.0", default-features = false, optional = true } tikv-jemallocator = { version = "0.5.4", default-features = false, optional = true } -tokio-postgres = { version = "0.7.7", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } +tokio-postgres = { version = "0.7.9", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = {version = "0.20.0", default-features = false, features = ["connect"], optional = true} toml = { version = "0.7.6", default-features = false, features = ["parse", "display"] } tonic = { version = "0.9", optional = true, default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index a2f2fa99215cc..548f62740b012 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -619,6 +619,7 @@ webbrowser,https://github.com/amodm/webbrowser-rs,MIT OR Apache-2.0,Amod Malviya webpki,https://github.com/briansmith/webpki,ISC,Brian Smith webpki-roots,https://github.com/rustls/webpki-roots,MPL-2.0,Joseph Birr-Pixton wepoll-ffi,https://github.com/aclysma/wepoll-ffi,MIT OR Apache-2.0 OR BSD-2-Clause,Philip Degarmo +whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,Kathryn Long widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,The widestring Authors winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian From d41f9f6082d64f1ea9585ae4642cd9646d8621b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 19:56:38 +0000 Subject: [PATCH 06/12] chore(deps): Bump h2 from 0.3.20 to 0.3.21 (#18330) Bumps [h2](https://github.com/hyperium/h2) from 0.3.20 to 0.3.21. - [Release notes](https://github.com/hyperium/h2/releases) - [Changelog](https://github.com/hyperium/h2/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/h2/compare/v0.3.20...v0.3.21) --- updated-dependencies: - dependency-name: h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7aae976ecea4..05d5925adba9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3807,9 +3807,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes 1.4.0", "fnv", diff --git a/Cargo.toml b/Cargo.toml index 73be098427db7..20532665b4c19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -269,7 +269,7 @@ futures-util = { version = "0.3.28", default-features = false } glob = { version = "0.3.1", default-features = false } governor = { version = "0.6.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } grok = { version = "2.0.0", default-features = false, optional = true } -h2 = { version = "0.3.20", default-features = false, optional = true } +h2 = { version = "0.3.21", default-features = false, optional = true } hash_hasher = { version = "2.0.0", default-features = false } hashbrown = { version = "0.14.0", default-features = false, optional = true, features = ["ahash"] } headers = { version = "0.3.8", default-features = false } From 6397edbcca5a0672985b93c8720dbe9c04caec8f Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Mon, 21 Aug 2023 14:04:27 -0600 Subject: [PATCH 07/12] chore(deps): Fix issue with `cargo` refetching refs on every run (#18331) bug(deps): Fix issue with `cargo` refetching refs on every run Cargo appears to have an issue with tag names containing a `+` that causes it to re-fetch the git repos as well as the cargo.io index on each run. Changing those tag names resolves the issue. --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05d5925adba9d..3b60529e4be98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5844,7 +5844,7 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" version = "300.1.3+3.1.2" -source = "git+https://github.com/vectordotdev/openssl-src-rs.git?tag=release-300-force-engine+3.1.2#98b1172bcef15358ad7bbf4baa3a3aa59d831e81" +source = "git+https://github.com/vectordotdev/openssl-src-rs?tag=release-300-force-engine_3.1.2#98b1172bcef15358ad7bbf4baa3a3aa59d831e81" dependencies = [ "cc", ] @@ -5852,7 +5852,7 @@ dependencies = [ [[package]] name = "openssl-sys" version = "0.9.91" -source = "git+https://github.com/vectordotdev/rust-openssl.git?tag=openssl-sys-v0.9.91+3.0.0#c3a8b494e0a8ab88db692c239d30c903353b56a3" +source = "git+https://github.com/vectordotdev/rust-openssl?tag=openssl-sys-v0.9.91_3.0.0#c3a8b494e0a8ab88db692c239d30c903353b56a3" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 20532665b4c19..17a62c31a20d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -391,8 +391,8 @@ ntapi = { git = "https://github.com/MSxDOS/ntapi.git", rev = "24fc1e47677fc9f6e3 # The current `openssl-sys` crate will vendor the OpenSSL sources via # `openssl-src` at version 1.1.1*, but we want version 3.1.*. Bring in forked # version of that crate with the appropriate dependency patched in. -openssl-sys = { git = "https://github.com/vectordotdev/rust-openssl.git", tag = "openssl-sys-v0.9.91+3.0.0" } -openssl-src = { git = "https://github.com/vectordotdev/openssl-src-rs.git", tag = "release-300-force-engine+3.1.2"} +openssl-sys = { git = "https://github.com/vectordotdev/rust-openssl", tag = "openssl-sys-v0.9.91_3.0.0" } +openssl-src = { git = "https://github.com/vectordotdev/openssl-src-rs", tag = "release-300-force-engine_3.1.2" } [features] # Default features for *-unknown-linux-gnu and *-apple-darwin From 71343bd91ee6e851b430c69ac27753ba0e41104c Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Mon, 21 Aug 2023 21:27:56 +0100 Subject: [PATCH 08/12] feat(route transform): Add option to enable/disable unmatched output (#18309) * feat(route transform): Add option to enable/disable unmatched output This commit adds a new boolean option `reroute_unmatched` to the `route` transform. It is inspired on the `reroute_dropped` option in the `remap` transform, allowing the user to control if they want or not the `._unmatched` output to be created and used. For backwards compatibility, this new option defaults to `true`. Users that are not interested in processing unmatched events can use this option to avoid the following warning on Vector startup, and also to remove the unnecessary `_unmatched` output in `vector top`: ``` WARN vector::config::loading: Transform "route._unmatched" has no consumers ``` Signed-off-by: Hugo Hromic * Update existing `can_serialize_remap` test * Add test for the new `reroute_unmatched` option --------- Signed-off-by: Hugo Hromic --- src/transforms/route.rs | 77 ++++++++++++++++--- .../components/transforms/base/route.cue | 45 +++++++---- 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/src/transforms/route.rs b/src/transforms/route.rs index e410277914a8f..721d92a9789ac 100644 --- a/src/transforms/route.rs +++ b/src/transforms/route.rs @@ -19,6 +19,7 @@ pub(crate) const UNMATCHED_ROUTE: &str = "_unmatched"; #[derive(Clone)] pub struct Route { conditions: Vec<(String, Condition)>, + reroute_unmatched: bool, } impl Route { @@ -28,7 +29,10 @@ impl Route { let condition = condition.build(&context.enrichment_tables)?; conditions.push((output_name.clone(), condition)); } - Ok(Self { conditions }) + Ok(Self { + conditions, + reroute_unmatched: config.reroute_unmatched, + }) } } @@ -47,7 +51,7 @@ impl SyncTransform for Route { check_failed += 1; } } - if check_failed == self.conditions.len() { + if self.reroute_unmatched && check_failed == self.conditions.len() { output.push(Some(UNMATCHED_ROUTE), event); } } @@ -61,11 +65,24 @@ impl SyncTransform for Route { #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct RouteConfig { + /// Reroutes unmatched events to a named output instead of silently discarding them. + /// + /// Normally, if an event doesn't match any defined route, it is sent to the `._unmatched` + /// output for further processing. In some cases, you may want to simply discard unmatched events and not + /// process them any further. + /// + /// In these cases, `reroute_unmatched` can be set to `false` to disable the `._unmatched` + /// output and instead silently discard any unmatched events. + #[serde(default = "crate::serde::default_true")] + #[configurable(metadata(docs::human_name = "Reroute Unmatched Events"))] + reroute_unmatched: bool, + /// A table of route identifiers to logical conditions representing the filter of the route. /// /// Each route can then be referenced as an input by other components with the name - /// `.`. If an event doesn’t match any route, it is sent to the - /// `._unmatched` output. + /// `.`. If an event doesn’t match any route, and if `reroute_unmatched` + /// is set to `true` (the default), it is sent to the `._unmatched` output. + /// Otherwise, the unmatched event is instead silently discarded. /// /// Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used /// as a route name. @@ -76,6 +93,7 @@ pub struct RouteConfig { impl GenerateConfig for RouteConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { + reroute_unmatched: true, route: IndexMap::new(), }) .unwrap() @@ -118,10 +136,12 @@ impl TransformConfig for RouteConfig { .with_port(output_name) }) .collect(); - result.push( - TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions)) - .with_port(UNMATCHED_ROUTE), - ); + if self.reroute_unmatched { + result.push( + TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions)) + .with_port(UNMATCHED_ROUTE), + ); + } result } @@ -162,7 +182,7 @@ mod test { assert_eq!( serde_json::to_string(&config).unwrap(), - r#"{"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"# + r#"{"reroute_unmatched":true,"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"# ); } @@ -293,6 +313,45 @@ mod test { } } + #[test] + fn route_no_unmatched_output() { + let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE]; + let event = Event::try_from(serde_json::json!({"message": "NOPE"})).unwrap(); + let config = toml::from_str::( + r#" + reroute_unmatched = false + + route.first.type = "vrl" + route.first.source = '.message == "hello world"' + + route.second.type = "vrl" + route.second.source = '.second == "second"' + + route.third.type = "vrl" + route.third.source = '.third == "third"' + "#, + ) + .unwrap(); + + let mut transform = Route::new(&config, &Default::default()).unwrap(); + let mut outputs = TransformOutputsBuf::new_with_capacity( + output_names + .iter() + .map(|output_name| { + TransformOutput::new(DataType::all(), HashMap::new()) + .with_port(output_name.to_owned()) + }) + .collect(), + 1, + ); + + transform.transform(event.clone(), &mut outputs); + for output_name in output_names { + let events: Vec<_> = outputs.drain_named(output_name).collect(); + assert_eq!(events.len(), 0); + } + } + #[tokio::test] async fn route_metrics_with_output_tag() { init_test(); diff --git a/website/cue/reference/components/transforms/base/route.cue b/website/cue/reference/components/transforms/base/route.cue index 873757ec82411..ad6e02580c813 100644 --- a/website/cue/reference/components/transforms/base/route.cue +++ b/website/cue/reference/components/transforms/base/route.cue @@ -1,20 +1,37 @@ package metadata -base: components: transforms: route: configuration: route: { - description: """ - A table of route identifiers to logical conditions representing the filter of the route. +base: components: transforms: route: configuration: { + reroute_unmatched: { + description: """ + Reroutes unmatched events to a named output instead of silently discarding them. - Each route can then be referenced as an input by other components with the name - `.`. If an event doesn’t match any route, it is sent to the - `._unmatched` output. + Normally, if an event doesn't match any defined route, it is sent to the `._unmatched` + output for further processing. In some cases, you may want to simply discard unmatched events and not + process them any further. - Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used - as a route name. - """ - required: false - type: object: options: "*": { - description: "An individual route." - required: true - type: condition: {} + In these cases, `reroute_unmatched` can be set to `false` to disable the `._unmatched` + output and instead silently discard any unmatched events. + """ + required: false + type: bool: default: true + } + route: { + description: """ + A table of route identifiers to logical conditions representing the filter of the route. + + Each route can then be referenced as an input by other components with the name + `.`. If an event doesn’t match any route, and if `reroute_unmatched` + is set to `true` (the default), it is sent to the `._unmatched` output. + Otherwise, the unmatched event is instead silently discarded. + + Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used + as a route name. + """ + required: false + type: object: options: "*": { + description: "An individual route." + required: true + type: condition: {} + } } } From 61c0ae8a54ecd17a4457f6916987effa3d2f903b Mon Sep 17 00:00:00 2001 From: Noemi <45180344+unflxw@users.noreply.github.com> Date: Mon, 21 Aug 2023 23:46:34 +0200 Subject: [PATCH 09/12] feat(appsignal sink): Normalize metrics (#18217) * feat(appsignal sink): Normalize metrics Implement a normaliser for the AppSignal sink to convert absolute counter metrics to incremental counters, and incremental gauges to absolute gauges. The AppSignal API ignores absolute counters and incremental gauges, so this change adds support for absolute counters and incremental gauges. This normaliser is inspired by the DataDog normaliser. * Refactor metric normalizer tests Move the methods that generate test metrics and the code that compares metric inputs and normalized outputs to the `test_util` metrics module. Previously these test helpers were duplicated across DataDog, StatsD and AppSignal's metric sinks. Rename the `run_comparisons` method to `assert_normalize`, as it asserts the results of running a normalizer's `.normalize` method. Move the duplicated test implementations to the `test_util::metrics` module, in a separate tests sub-module, and make them generic over the normalizer. Use these test definitions in the DataDog, StatsD and AppSignal's metric sink tests. * Fix integration tests Since the AppSignal sink now normalises counters from absolute to incremental, absolute counters that are only emitted once do not result in an outgoing HTTP request being emitted by the sink. Address this by emitting the absolute counters in the tests at least twice. This also implicitly tests the metrics' normalisation. --- src/sinks/appsignal/integration_tests.rs | 52 ++-- src/sinks/appsignal/mod.rs | 1 + src/sinks/appsignal/normalizer.rs | 78 ++++++ src/sinks/appsignal/sink.rs | 15 +- src/sinks/datadog/metrics/normalizer.rs | 303 ++-------------------- src/sinks/statsd/normalizer.rs | 266 ++----------------- src/test_util/metrics.rs | 313 ++++++++++++++++++++++- 7 files changed, 476 insertions(+), 552 deletions(-) create mode 100644 src/sinks/appsignal/normalizer.rs diff --git a/src/sinks/appsignal/integration_tests.rs b/src/sinks/appsignal/integration_tests.rs index 1c28032caf40d..8054da4eb92cc 100644 --- a/src/sinks/appsignal/integration_tests.rs +++ b/src/sinks/appsignal/integration_tests.rs @@ -98,14 +98,23 @@ async fn metrics_real_endpoint() { #[tokio::test] async fn metrics_shape() { let events: Vec<_> = (0..5) - .map(|index| { - Event::Metric(Metric::new( - format!("counter_{}", index), - MetricKind::Absolute, - MetricValue::Counter { - value: index as f64, - }, - )) + .flat_map(|index| { + vec![ + Event::Metric(Metric::new( + format!("counter_{}", index), + MetricKind::Absolute, + MetricValue::Counter { + value: index as f64, + }, + )), + Event::Metric(Metric::new( + format!("counter_{}", index), + MetricKind::Absolute, + MetricValue::Counter { + value: (index + index) as f64, + }, + )), + ] }) .collect(); let api_key = push_api_key(); @@ -146,11 +155,11 @@ async fn metrics_shape() { .collect(); assert_eq!( vec![ - ("counter_0", "absolute", 0.0), - ("counter_1", "absolute", 1.0), - ("counter_2", "absolute", 2.0), - ("counter_3", "absolute", 3.0), - ("counter_4", "absolute", 4.0), + ("counter_0", "incremental", 0.0), + ("counter_1", "incremental", 1.0), + ("counter_2", "incremental", 2.0), + ("counter_3", "incremental", 3.0), + ("counter_4", "incremental", 4.0), ], metrics ); @@ -231,11 +240,18 @@ async fn error_scenario_real_endpoint() { let (sink, _) = config.build(cx).await.unwrap(); let (batch, receiver) = BatchNotifier::new_with_receiver(); - let events = vec![Event::Metric(Metric::new( - "counter", - MetricKind::Absolute, - MetricValue::Counter { value: 1.0 }, - ))]; + let events = vec![ + Event::Metric(Metric::new( + "counter", + MetricKind::Absolute, + MetricValue::Counter { value: 1.0 }, + )), + Event::Metric(Metric::new( + "counter", + MetricKind::Absolute, + MetricValue::Counter { value: 2.0 }, + )), + ]; let stream = map_event_batch_stream(stream::iter(events.clone()), Some(batch)); sink.run(stream).await.unwrap(); diff --git a/src/sinks/appsignal/mod.rs b/src/sinks/appsignal/mod.rs index 175c456bf6cbc..ecfa404b33dbf 100644 --- a/src/sinks/appsignal/mod.rs +++ b/src/sinks/appsignal/mod.rs @@ -8,6 +8,7 @@ mod config; mod encoder; +mod normalizer; mod request_builder; mod service; mod sink; diff --git a/src/sinks/appsignal/normalizer.rs b/src/sinks/appsignal/normalizer.rs new file mode 100644 index 0000000000000..0108de6516b3f --- /dev/null +++ b/src/sinks/appsignal/normalizer.rs @@ -0,0 +1,78 @@ +use vector_core::event::{Metric, MetricValue}; + +use crate::sinks::util::buffer::metrics::{MetricNormalize, MetricSet}; + +#[derive(Default)] +pub(crate) struct AppsignalMetricsNormalizer; + +impl MetricNormalize for AppsignalMetricsNormalizer { + fn normalize(&mut self, state: &mut MetricSet, metric: Metric) -> Option { + // We only care about making sure that counters are incremental, and that gauges are + // always absolute. Other metric types are currently unsupported. + match &metric.value() { + // We always send counters as incremental and gauges as absolute. Realistically, any + // system sending an incremental gauge update is kind of doing it wrong, but alas. + MetricValue::Counter { .. } => state.make_incremental(metric), + MetricValue::Gauge { .. } => state.make_absolute(metric), + // Otherwise, send it through as-is. + _ => Some(metric), + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use crate::event::{Metric, MetricKind, MetricValue}; + + use super::AppsignalMetricsNormalizer; + use crate::test_util::metrics::{assert_normalize, tests}; + + #[test] + fn absolute_counter() { + tests::absolute_counter_normalize_to_incremental(AppsignalMetricsNormalizer); + } + + #[test] + fn incremental_counter() { + tests::incremental_counter_normalize_to_incremental(AppsignalMetricsNormalizer); + } + + #[test] + fn mixed_counter() { + tests::mixed_counter_normalize_to_incremental(AppsignalMetricsNormalizer); + } + + #[test] + fn absolute_gauge() { + tests::absolute_gauge_normalize_to_absolute(AppsignalMetricsNormalizer); + } + + #[test] + fn incremental_gauge() { + tests::incremental_gauge_normalize_to_absolute(AppsignalMetricsNormalizer); + } + + #[test] + fn mixed_gauge() { + tests::mixed_gauge_normalize_to_absolute(AppsignalMetricsNormalizer); + } + + #[test] + fn other_metrics() { + let metric = Metric::new( + "set", + MetricKind::Incremental, + MetricValue::Set { + values: BTreeSet::new(), + }, + ); + + assert_normalize( + AppsignalMetricsNormalizer, + vec![metric.clone()], + vec![Some(metric)], + ); + } +} diff --git a/src/sinks/appsignal/sink.rs b/src/sinks/appsignal/sink.rs index ab9b135829abb..b908ed757a45a 100644 --- a/src/sinks/appsignal/sink.rs +++ b/src/sinks/appsignal/sink.rs @@ -1,4 +1,5 @@ use futures::{stream::BoxStream, StreamExt}; +use futures_util::future::ready; use tower::{Service, ServiceBuilder}; use vector_core::{ event::Event, @@ -7,12 +8,14 @@ use vector_core::{ }; use crate::{ - codecs::Transformer, internal_events::SinkRequestBuildError, - sinks::util::builder::SinkBuilderExt, sinks::util::Compression, + codecs::Transformer, + internal_events::SinkRequestBuildError, + sinks::util::{buffer::metrics::MetricNormalizer, builder::SinkBuilderExt, Compression}, }; use super::{ encoder::AppsignalEncoder, + normalizer::AppsignalMetricsNormalizer, request_builder::{AppsignalRequest, AppsignalRequestBuilder}, }; @@ -32,8 +35,16 @@ where { pub(super) async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { let service = ServiceBuilder::new().service(self.service); + let mut normalizer = MetricNormalizer::::default(); input + .filter_map(move |event| { + ready(if let Event::Metric(metric) = event { + normalizer.normalize(metric).map(Event::Metric) + } else { + Some(event) + }) + }) .batched(self.batch_settings.into_byte_size_config()) .request_builder( None, diff --git a/src/sinks/datadog/metrics/normalizer.rs b/src/sinks/datadog/metrics/normalizer.rs index d4e6430250e1c..6b6d27b1c0649 100644 --- a/src/sinks/datadog/metrics/normalizer.rs +++ b/src/sinks/datadog/metrics/normalizer.rs @@ -38,131 +38,16 @@ impl MetricNormalize for DatadogMetricsNormalizer { #[cfg(test)] mod tests { - use std::fmt; - use vector_core::{ - event::{ - metric::{Bucket, MetricSketch, Sample}, - Metric, MetricKind, MetricValue, StatisticKind, - }, + event::{metric::MetricSketch, Metric, MetricKind, MetricValue}, metrics::AgentDDSketch, }; use super::DatadogMetricsNormalizer; - use crate::sinks::util::buffer::metrics::{MetricNormalize, MetricSet}; - - fn buckets_from_samples(values: &[f64]) -> (Vec, f64, u64) { - // Generate buckets, and general statistics, for an input set of data. We only use this in - // tests, and so we have some semi-realistic buckets here, but mainly we use them for testing, - // not for most accurately/efficiently representing the input samples. - let bounds = &[ - 1.0, - 2.0, - 4.0, - 8.0, - 16.0, - 32.0, - 64.0, - 128.0, - 256.0, - 512.0, - 1024.0, - f64::INFINITY, - ]; - let mut buckets = bounds - .iter() - .map(|b| Bucket { - upper_limit: *b, - count: 0, - }) - .collect::>(); - - let mut sum = 0.0; - let mut count = 0; - for value in values { - for bucket in buckets.iter_mut() { - if *value <= bucket.upper_limit { - bucket.count += 1; - } - } - - sum += *value; - count += 1; - } - - (buckets, sum, count) - } - - fn generate_f64s(start: u16, end: u16) -> Vec { - assert!(start <= end); - let mut samples = Vec::new(); - for n in start..=end { - samples.push(f64::from(n)); - } - samples - } - - fn get_counter(value: f64, kind: MetricKind) -> Metric { - Metric::new("counter", kind, MetricValue::Counter { value }) - } - - fn get_gauge(value: f64, kind: MetricKind) -> Metric { - Metric::new("gauge", kind, MetricValue::Gauge { value }) - } - fn get_set(values: S, kind: MetricKind) -> Metric - where - S: IntoIterator, - V: fmt::Display, - { - Metric::new( - "set", - kind, - MetricValue::Set { - values: values.into_iter().map(|i| i.to_string()).collect(), - }, - ) - } - - fn get_distribution(samples: S, kind: MetricKind) -> Metric - where - S: IntoIterator, - V: Into, - { - Metric::new( - "distribution", - kind, - MetricValue::Distribution { - samples: samples - .into_iter() - .map(|n| Sample { - value: n.into(), - rate: 1, - }) - .collect(), - statistic: StatisticKind::Histogram, - }, - ) - } - - fn get_aggregated_histogram(samples: S, kind: MetricKind) -> Metric - where - S: IntoIterator, - V: Into, - { - let samples = samples.into_iter().map(Into::into).collect::>(); - let (buckets, sum, count) = buckets_from_samples(&samples); - - Metric::new( - "agg_histogram", - kind, - MetricValue::AggregatedHistogram { - buckets, - count, - sum, - }, - ) - } + use crate::test_util::metrics::{ + assert_normalize, generate_f64s, get_aggregated_histogram, get_distribution, tests, + }; fn get_sketch(name: N, samples: S, kind: MetricKind) -> Metric where @@ -183,199 +68,49 @@ mod tests { ) } - fn run_comparisons(inputs: Vec, expected_outputs: Vec>) { - let mut metric_set = MetricSet::default(); - let mut normalizer = DatadogMetricsNormalizer; - - for (input, expected) in inputs.into_iter().zip(expected_outputs) { - let result = normalizer.normalize(&mut metric_set, input); - assert_eq!(result, expected); - } - } - #[test] fn absolute_counter() { - let first_value = 3.14; - let second_value = 8.675309; - - let counters = vec![ - get_counter(first_value, MetricKind::Absolute), - get_counter(second_value, MetricKind::Absolute), - ]; - - let expected_counters = vec![ - None, - Some(get_counter( - second_value - first_value, - MetricKind::Incremental, - )), - ]; - - run_comparisons(counters, expected_counters); + tests::absolute_counter_normalize_to_incremental(DatadogMetricsNormalizer); } #[test] fn incremental_counter() { - let first_value = 3.14; - let second_value = 8.675309; - - let counters = vec![ - get_counter(first_value, MetricKind::Incremental), - get_counter(second_value, MetricKind::Incremental), - ]; - - let expected_counters = counters - .clone() - .into_iter() - .map(Option::Some) - .collect::>(); - - run_comparisons(counters, expected_counters); + tests::incremental_counter_normalize_to_incremental(DatadogMetricsNormalizer); } #[test] fn mixed_counter() { - let first_value = 3.14; - let second_value = 8.675309; - let third_value = 16.19; - - let counters = vec![ - get_counter(first_value, MetricKind::Incremental), - get_counter(second_value, MetricKind::Absolute), - get_counter(third_value, MetricKind::Absolute), - get_counter(first_value, MetricKind::Absolute), - get_counter(second_value, MetricKind::Incremental), - get_counter(third_value, MetricKind::Incremental), - ]; - - let expected_counters = vec![ - Some(get_counter(first_value, MetricKind::Incremental)), - None, - Some(get_counter( - third_value - second_value, - MetricKind::Incremental, - )), - None, - Some(get_counter(second_value, MetricKind::Incremental)), - Some(get_counter(third_value, MetricKind::Incremental)), - ]; - - run_comparisons(counters, expected_counters); + tests::mixed_counter_normalize_to_incremental(DatadogMetricsNormalizer); } #[test] fn absolute_gauge() { - let first_value = 3.14; - let second_value = 8.675309; - - let gauges = vec![ - get_gauge(first_value, MetricKind::Absolute), - get_gauge(second_value, MetricKind::Absolute), - ]; - - let expected_gauges = gauges - .clone() - .into_iter() - .map(Option::Some) - .collect::>(); - - run_comparisons(gauges, expected_gauges); + tests::absolute_gauge_normalize_to_absolute(DatadogMetricsNormalizer); } #[test] fn incremental_gauge() { - let first_value = 3.14; - let second_value = 8.675309; - - let gauges = vec![ - get_gauge(first_value, MetricKind::Incremental), - get_gauge(second_value, MetricKind::Incremental), - ]; - - let expected_gauges = vec![ - Some(get_gauge(first_value, MetricKind::Absolute)), - Some(get_gauge(first_value + second_value, MetricKind::Absolute)), - ]; - - run_comparisons(gauges, expected_gauges); + tests::incremental_gauge_normalize_to_absolute(DatadogMetricsNormalizer); } #[test] fn mixed_gauge() { - let first_value = 3.14; - let second_value = 8.675309; - let third_value = 16.19; - - let gauges = vec![ - get_gauge(first_value, MetricKind::Incremental), - get_gauge(second_value, MetricKind::Absolute), - get_gauge(third_value, MetricKind::Absolute), - get_gauge(first_value, MetricKind::Absolute), - get_gauge(second_value, MetricKind::Incremental), - get_gauge(third_value, MetricKind::Incremental), - ]; - - let expected_gauges = vec![ - Some(get_gauge(first_value, MetricKind::Absolute)), - Some(get_gauge(second_value, MetricKind::Absolute)), - Some(get_gauge(third_value, MetricKind::Absolute)), - Some(get_gauge(first_value, MetricKind::Absolute)), - Some(get_gauge(first_value + second_value, MetricKind::Absolute)), - Some(get_gauge( - first_value + second_value + third_value, - MetricKind::Absolute, - )), - ]; - - run_comparisons(gauges, expected_gauges); + tests::mixed_gauge_normalize_to_absolute(DatadogMetricsNormalizer); } #[test] fn absolute_set() { - let sets = vec![ - get_set(1..=20, MetricKind::Absolute), - get_set(15..=25, MetricKind::Absolute), - ]; - - let expected_sets = vec![None, Some(get_set(21..=25, MetricKind::Incremental))]; - - run_comparisons(sets, expected_sets); + tests::absolute_set_normalize_to_incremental(DatadogMetricsNormalizer); } #[test] fn incremental_set() { - let sets = vec![ - get_set(1..=20, MetricKind::Incremental), - get_set(15..=25, MetricKind::Incremental), - ]; - - let expected_sets = vec![ - Some(get_set(1..=20, MetricKind::Incremental)), - Some(get_set(15..=25, MetricKind::Incremental)), - ]; - - run_comparisons(sets, expected_sets); + tests::incremental_set_normalize_to_incremental(DatadogMetricsNormalizer); } #[test] fn mixed_set() { - let sets = vec![ - get_set(1..=20, MetricKind::Incremental), - get_set(10..=16, MetricKind::Absolute), - get_set(15..=25, MetricKind::Absolute), - get_set(1..5, MetricKind::Incremental), - get_set(3..=42, MetricKind::Incremental), - ]; - - let expected_sets = vec![ - Some(get_set(1..=20, MetricKind::Incremental)), - None, - Some(get_set(17..=25, MetricKind::Incremental)), - Some(get_set(1..5, MetricKind::Incremental)), - Some(get_set(3..=42, MetricKind::Incremental)), - ]; - - run_comparisons(sets, expected_sets); + tests::mixed_set_normalize_to_incremental(DatadogMetricsNormalizer); } #[test] @@ -401,7 +136,7 @@ mod tests { )), ]; - run_comparisons(distributions, expected_sketches); + assert_normalize(DatadogMetricsNormalizer, distributions, expected_sketches); } #[test] @@ -429,7 +164,7 @@ mod tests { )), ]; - run_comparisons(distributions, expected_sketches); + assert_normalize(DatadogMetricsNormalizer, distributions, expected_sketches); } #[test] @@ -476,7 +211,7 @@ mod tests { )), ]; - run_comparisons(distributions, expected_sketches); + assert_normalize(DatadogMetricsNormalizer, distributions, expected_sketches); } #[test] @@ -501,7 +236,7 @@ mod tests { ), ]; - run_comparisons(agg_histograms, expected_sketches); + assert_normalize(DatadogMetricsNormalizer, agg_histograms, expected_sketches); } #[test] @@ -533,7 +268,7 @@ mod tests { ), ]; - run_comparisons(agg_histograms, expected_sketches); + assert_normalize(DatadogMetricsNormalizer, agg_histograms, expected_sketches); } #[test] @@ -588,6 +323,6 @@ mod tests { ), ]; - run_comparisons(agg_histograms, expected_sketches); + assert_normalize(DatadogMetricsNormalizer, agg_histograms, expected_sketches); } } diff --git a/src/sinks/statsd/normalizer.rs b/src/sinks/statsd/normalizer.rs index e497d074dfb16..2592292e7023c 100644 --- a/src/sinks/statsd/normalizer.rs +++ b/src/sinks/statsd/normalizer.rs @@ -20,226 +20,33 @@ impl MetricNormalize for StatsdNormalizer { #[cfg(test)] mod tests { - use std::fmt; - - use vector_core::event::{ - metric::{Bucket, Sample}, - Metric, MetricKind, MetricValue, StatisticKind, - }; + use vector_core::event::MetricKind; use super::StatsdNormalizer; - use crate::sinks::util::buffer::metrics::{MetricNormalize, MetricSet}; - - fn buckets_from_samples(values: &[f64]) -> (Vec, f64, u64) { - // Generate buckets, and general statistics, for an input set of data. We only use this in - // tests, and so we have some semi-realistic buckets here, but mainly we use them for testing, - // not for most accurately/efficiently representing the input samples. - let bounds = &[ - 1.0, - 2.0, - 4.0, - 8.0, - 16.0, - 32.0, - 64.0, - 128.0, - 256.0, - 512.0, - 1024.0, - f64::INFINITY, - ]; - let mut buckets = bounds - .iter() - .map(|b| Bucket { - upper_limit: *b, - count: 0, - }) - .collect::>(); - - let mut sum = 0.0; - let mut count = 0; - for value in values { - for bucket in buckets.iter_mut() { - if *value <= bucket.upper_limit { - bucket.count += 1; - } - } - - sum += *value; - count += 1; - } - - (buckets, sum, count) - } - - fn generate_f64s(start: u16, end: u16) -> Vec { - assert!(start <= end); - let mut samples = Vec::new(); - for n in start..=end { - samples.push(f64::from(n)); - } - samples - } - - fn get_counter(value: f64, kind: MetricKind) -> Metric { - Metric::new("counter", kind, MetricValue::Counter { value }) - } - - fn get_gauge(value: f64, kind: MetricKind) -> Metric { - Metric::new("gauge", kind, MetricValue::Gauge { value }) - } - - fn get_set(values: S, kind: MetricKind) -> Metric - where - S: IntoIterator, - V: fmt::Display, - { - Metric::new( - "set", - kind, - MetricValue::Set { - values: values.into_iter().map(|i| i.to_string()).collect(), - }, - ) - } - - fn get_distribution(samples: S, kind: MetricKind) -> Metric - where - S: IntoIterator, - V: Into, - { - Metric::new( - "distribution", - kind, - MetricValue::Distribution { - samples: samples - .into_iter() - .map(|n| Sample { - value: n.into(), - rate: 1, - }) - .collect(), - statistic: StatisticKind::Histogram, - }, - ) - } - - fn get_aggregated_histogram(samples: S, kind: MetricKind) -> Metric - where - S: IntoIterator, - V: Into, - { - let samples = samples.into_iter().map(Into::into).collect::>(); - let (buckets, sum, count) = buckets_from_samples(&samples); - - Metric::new( - "agg_histogram", - kind, - MetricValue::AggregatedHistogram { - buckets, - count, - sum, - }, - ) - } - fn run_comparisons(inputs: Vec, expected_outputs: Vec>) { - let mut metric_set = MetricSet::default(); - let mut normalizer = StatsdNormalizer; - - for (input, expected) in inputs.into_iter().zip(expected_outputs) { - let result = normalizer.normalize(&mut metric_set, input); - assert_eq!(result, expected); - } - } + use crate::test_util::metrics::{ + assert_normalize, generate_f64s, get_aggregated_histogram, get_distribution, get_gauge, + tests, + }; #[test] fn absolute_counter() { - let first_value = 3.14; - let second_value = 8.675309; - - let counters = vec![ - get_counter(first_value, MetricKind::Absolute), - get_counter(second_value, MetricKind::Absolute), - ]; - - let expected_counters = vec![ - None, - Some(get_counter( - second_value - first_value, - MetricKind::Incremental, - )), - ]; - - run_comparisons(counters, expected_counters); + tests::absolute_counter_normalize_to_incremental(StatsdNormalizer); } #[test] fn incremental_counter() { - let first_value = 3.14; - let second_value = 8.675309; - - let counters = vec![ - get_counter(first_value, MetricKind::Incremental), - get_counter(second_value, MetricKind::Incremental), - ]; - - let expected_counters = counters - .clone() - .into_iter() - .map(Option::Some) - .collect::>(); - - run_comparisons(counters, expected_counters); + tests::incremental_counter_normalize_to_incremental(StatsdNormalizer); } #[test] fn mixed_counter() { - let first_value = 3.14; - let second_value = 8.675309; - let third_value = 16.19; - - let counters = vec![ - get_counter(first_value, MetricKind::Incremental), - get_counter(second_value, MetricKind::Absolute), - get_counter(third_value, MetricKind::Absolute), - get_counter(first_value, MetricKind::Absolute), - get_counter(second_value, MetricKind::Incremental), - get_counter(third_value, MetricKind::Incremental), - ]; - - let expected_counters = vec![ - Some(get_counter(first_value, MetricKind::Incremental)), - None, - Some(get_counter( - third_value - second_value, - MetricKind::Incremental, - )), - None, - Some(get_counter(second_value, MetricKind::Incremental)), - Some(get_counter(third_value, MetricKind::Incremental)), - ]; - - run_comparisons(counters, expected_counters); + tests::mixed_counter_normalize_to_incremental(StatsdNormalizer); } #[test] fn absolute_gauge() { - let first_value = 3.14; - let second_value = 8.675309; - - let gauges = vec![ - get_gauge(first_value, MetricKind::Absolute), - get_gauge(second_value, MetricKind::Absolute), - ]; - - let expected_gauges = gauges - .clone() - .into_iter() - .map(Option::Some) - .collect::>(); - - run_comparisons(gauges, expected_gauges); + tests::absolute_gauge_normalize_to_absolute(StatsdNormalizer); } #[test] @@ -258,7 +65,7 @@ mod tests { .map(Option::Some) .collect::>(); - run_comparisons(gauges, expected_gauges); + assert_normalize(StatsdNormalizer, gauges, expected_gauges); } #[test] @@ -282,55 +89,22 @@ mod tests { .map(Option::Some) .collect::>(); - run_comparisons(gauges, expected_gauges); + assert_normalize(StatsdNormalizer, gauges, expected_gauges); } #[test] fn absolute_set() { - let sets = vec![ - get_set(1..=20, MetricKind::Absolute), - get_set(15..=25, MetricKind::Absolute), - ]; - - let expected_sets = vec![None, Some(get_set(21..=25, MetricKind::Incremental))]; - - run_comparisons(sets, expected_sets); + tests::absolute_set_normalize_to_incremental(StatsdNormalizer); } #[test] fn incremental_set() { - let sets = vec![ - get_set(1..=20, MetricKind::Incremental), - get_set(15..=25, MetricKind::Incremental), - ]; - - let expected_sets = vec![ - Some(get_set(1..=20, MetricKind::Incremental)), - Some(get_set(15..=25, MetricKind::Incremental)), - ]; - - run_comparisons(sets, expected_sets); + tests::incremental_set_normalize_to_incremental(StatsdNormalizer); } #[test] fn mixed_set() { - let sets = vec![ - get_set(1..=20, MetricKind::Incremental), - get_set(10..=16, MetricKind::Absolute), - get_set(15..=25, MetricKind::Absolute), - get_set(1..5, MetricKind::Incremental), - get_set(3..=42, MetricKind::Incremental), - ]; - - let expected_sets = vec![ - Some(get_set(1..=20, MetricKind::Incremental)), - None, - Some(get_set(17..=25, MetricKind::Incremental)), - Some(get_set(1..5, MetricKind::Incremental)), - Some(get_set(3..=42, MetricKind::Incremental)), - ]; - - run_comparisons(sets, expected_sets); + tests::mixed_set_normalize_to_incremental(StatsdNormalizer); } #[test] @@ -349,7 +123,7 @@ mod tests { Some(get_distribution(expected_samples, MetricKind::Incremental)), ]; - run_comparisons(distributions, expected_distributions); + assert_normalize(StatsdNormalizer, distributions, expected_distributions); } #[test] @@ -364,7 +138,7 @@ mod tests { let expected_distributions = distributions.iter().cloned().map(Some).collect(); - run_comparisons(distributions, expected_distributions); + assert_normalize(StatsdNormalizer, distributions, expected_distributions); } #[test] @@ -394,7 +168,7 @@ mod tests { Some(distributions[4].clone()), ]; - run_comparisons(distributions, expected_distributions); + assert_normalize(StatsdNormalizer, distributions, expected_distributions); } #[test] @@ -409,7 +183,7 @@ mod tests { let expected_agg_histograms = vec![]; - run_comparisons(agg_histograms, expected_agg_histograms); + assert_normalize(StatsdNormalizer, agg_histograms, expected_agg_histograms); } #[test] @@ -428,7 +202,7 @@ mod tests { .map(Option::Some) .collect::>(); - run_comparisons(agg_histograms, expected_agg_histograms); + assert_normalize(StatsdNormalizer, agg_histograms, expected_agg_histograms); } #[test] @@ -449,6 +223,6 @@ mod tests { let expected_agg_histograms = vec![]; - run_comparisons(agg_histograms, expected_agg_histograms); + assert_normalize(StatsdNormalizer, agg_histograms, expected_agg_histograms); } } diff --git a/src/test_util/metrics.rs b/src/test_util/metrics.rs index 9a3bdca726b71..749df9d736ce9 100644 --- a/src/test_util/metrics.rs +++ b/src/test_util/metrics.rs @@ -1,10 +1,13 @@ use std::collections::{HashMap, HashSet}; +use std::fmt::Display; + use vector_core::event::{ - metric::{MetricData, MetricSeries, Sample}, - Event, EventMetadata, Metric, MetricValue, + metric::{Bucket, MetricData, MetricSeries, Sample}, + Event, EventMetadata, Metric, MetricValue, StatisticKind, }; +use crate::event::MetricKind; use crate::sinks::util::buffer::metrics::{MetricNormalize, MetricSet}; type SplitMetrics = HashMap; @@ -246,3 +249,309 @@ pub fn assert_set(metrics: &SplitMetrics, series: MetricSeries, expected_values: assert_eq!(actual_values, expected_values); } + +fn buckets_from_samples(values: &[f64]) -> (Vec, f64, u64) { + // Generate buckets, and general statistics, for an input set of data. We only use this in + // tests, and so we have some semi-realistic buckets here, but mainly we use them for testing, + // not for most accurately/efficiently representing the input samples. + let bounds = &[ + 1.0, + 2.0, + 4.0, + 8.0, + 16.0, + 32.0, + 64.0, + 128.0, + 256.0, + 512.0, + 1024.0, + f64::INFINITY, + ]; + let mut buckets = bounds + .iter() + .map(|b| Bucket { + upper_limit: *b, + count: 0, + }) + .collect::>(); + + let mut sum = 0.0; + let mut count = 0; + for value in values { + for bucket in buckets.iter_mut() { + if *value <= bucket.upper_limit { + bucket.count += 1; + } + } + + sum += *value; + count += 1; + } + + (buckets, sum, count) +} + +pub fn generate_f64s(start: u16, end: u16) -> Vec { + assert!(start <= end); + let mut samples = Vec::new(); + for n in start..=end { + samples.push(f64::from(n)); + } + samples +} + +pub fn get_set(values: S, kind: MetricKind) -> Metric +where + S: IntoIterator, + V: Display, +{ + Metric::new( + "set", + kind, + MetricValue::Set { + values: values.into_iter().map(|i| i.to_string()).collect(), + }, + ) +} + +pub fn get_distribution(samples: S, kind: MetricKind) -> Metric +where + S: IntoIterator, + V: Into, +{ + Metric::new( + "distribution", + kind, + MetricValue::Distribution { + samples: samples + .into_iter() + .map(|n| Sample { + value: n.into(), + rate: 1, + }) + .collect(), + statistic: StatisticKind::Histogram, + }, + ) +} + +pub fn get_aggregated_histogram(samples: S, kind: MetricKind) -> Metric +where + S: IntoIterator, + V: Into, +{ + let samples = samples.into_iter().map(Into::into).collect::>(); + let (buckets, sum, count) = buckets_from_samples(&samples); + + Metric::new( + "agg_histogram", + kind, + MetricValue::AggregatedHistogram { + buckets, + count, + sum, + }, + ) +} + +pub fn get_counter(value: f64, kind: MetricKind) -> Metric { + Metric::new("counter", kind, MetricValue::Counter { value }) +} + +pub fn get_gauge(value: f64, kind: MetricKind) -> Metric { + Metric::new("gauge", kind, MetricValue::Gauge { value }) +} + +pub fn assert_normalize( + mut normalizer: N, + inputs: Vec, + expected_outputs: Vec>, +) { + let mut metric_set = MetricSet::default(); + + for (input, expected) in inputs.into_iter().zip(expected_outputs) { + let result = normalizer.normalize(&mut metric_set, input); + assert_eq!(result, expected); + } +} + +pub mod tests { + use super::*; + + pub fn absolute_counter_normalize_to_incremental(normalizer: N) { + let first_value = 3.14; + let second_value = 8.675309; + + let counters = vec![ + get_counter(first_value, MetricKind::Absolute), + get_counter(second_value, MetricKind::Absolute), + ]; + + let expected_counters = vec![ + None, + Some(get_counter( + second_value - first_value, + MetricKind::Incremental, + )), + ]; + + assert_normalize(normalizer, counters, expected_counters); + } + + pub fn incremental_counter_normalize_to_incremental(normalizer: N) { + let first_value = 3.14; + let second_value = 8.675309; + + let counters = vec![ + get_counter(first_value, MetricKind::Incremental), + get_counter(second_value, MetricKind::Incremental), + ]; + + let expected_counters = counters + .clone() + .into_iter() + .map(Option::Some) + .collect::>(); + + assert_normalize(normalizer, counters, expected_counters); + } + + pub fn mixed_counter_normalize_to_incremental(normalizer: N) { + let first_value = 3.14; + let second_value = 8.675309; + let third_value = 16.19; + + let counters = vec![ + get_counter(first_value, MetricKind::Incremental), + get_counter(second_value, MetricKind::Absolute), + get_counter(third_value, MetricKind::Absolute), + get_counter(first_value, MetricKind::Absolute), + get_counter(second_value, MetricKind::Incremental), + get_counter(third_value, MetricKind::Incremental), + ]; + + let expected_counters = vec![ + Some(get_counter(first_value, MetricKind::Incremental)), + None, + Some(get_counter( + third_value - second_value, + MetricKind::Incremental, + )), + None, + Some(get_counter(second_value, MetricKind::Incremental)), + Some(get_counter(third_value, MetricKind::Incremental)), + ]; + + assert_normalize(normalizer, counters, expected_counters); + } + + pub fn absolute_gauge_normalize_to_absolute(normalizer: N) { + let first_value = 3.14; + let second_value = 8.675309; + + let gauges = vec![ + get_gauge(first_value, MetricKind::Absolute), + get_gauge(second_value, MetricKind::Absolute), + ]; + + let expected_gauges = gauges + .clone() + .into_iter() + .map(Option::Some) + .collect::>(); + + assert_normalize(normalizer, gauges, expected_gauges); + } + + pub fn incremental_gauge_normalize_to_absolute(normalizer: N) { + let first_value = 3.14; + let second_value = 8.675309; + + let gauges = vec![ + get_gauge(first_value, MetricKind::Incremental), + get_gauge(second_value, MetricKind::Incremental), + ]; + + let expected_gauges = vec![ + Some(get_gauge(first_value, MetricKind::Absolute)), + Some(get_gauge(first_value + second_value, MetricKind::Absolute)), + ]; + + assert_normalize(normalizer, gauges, expected_gauges); + } + + pub fn mixed_gauge_normalize_to_absolute(normalizer: N) { + let first_value = 3.14; + let second_value = 8.675309; + let third_value = 16.19; + + let gauges = vec![ + get_gauge(first_value, MetricKind::Incremental), + get_gauge(second_value, MetricKind::Absolute), + get_gauge(third_value, MetricKind::Absolute), + get_gauge(first_value, MetricKind::Absolute), + get_gauge(second_value, MetricKind::Incremental), + get_gauge(third_value, MetricKind::Incremental), + ]; + + let expected_gauges = vec![ + Some(get_gauge(first_value, MetricKind::Absolute)), + Some(get_gauge(second_value, MetricKind::Absolute)), + Some(get_gauge(third_value, MetricKind::Absolute)), + Some(get_gauge(first_value, MetricKind::Absolute)), + Some(get_gauge(first_value + second_value, MetricKind::Absolute)), + Some(get_gauge( + first_value + second_value + third_value, + MetricKind::Absolute, + )), + ]; + + assert_normalize(normalizer, gauges, expected_gauges); + } + + pub fn absolute_set_normalize_to_incremental(normalizer: N) { + let sets = vec![ + get_set(1..=20, MetricKind::Absolute), + get_set(15..=25, MetricKind::Absolute), + ]; + + let expected_sets = vec![None, Some(get_set(21..=25, MetricKind::Incremental))]; + + assert_normalize(normalizer, sets, expected_sets); + } + + pub fn incremental_set_normalize_to_incremental(normalizer: N) { + let sets = vec![ + get_set(1..=20, MetricKind::Incremental), + get_set(15..=25, MetricKind::Incremental), + ]; + + let expected_sets = vec![ + Some(get_set(1..=20, MetricKind::Incremental)), + Some(get_set(15..=25, MetricKind::Incremental)), + ]; + + assert_normalize(normalizer, sets, expected_sets); + } + + pub fn mixed_set_normalize_to_incremental(normalizer: N) { + let sets = vec![ + get_set(1..=20, MetricKind::Incremental), + get_set(10..=16, MetricKind::Absolute), + get_set(15..=25, MetricKind::Absolute), + get_set(1..5, MetricKind::Incremental), + get_set(3..=42, MetricKind::Incremental), + ]; + + let expected_sets = vec![ + Some(get_set(1..=20, MetricKind::Incremental)), + None, + Some(get_set(17..=25, MetricKind::Incremental)), + Some(get_set(1..5, MetricKind::Incremental)), + Some(get_set(3..=42, MetricKind::Incremental)), + ]; + + assert_normalize(normalizer, sets, expected_sets); + } +} From 85dd43921f9948148d088cb94d3e49127cd613c1 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 22 Aug 2023 10:55:55 -0400 Subject: [PATCH 10/12] chore(docs): make YAML appear first in the example configurations (#18325) --- .../architecture/pipeline-model.md | 2 +- .../en/docs/reference/configuration/_index.md | 2 +- website/cue/reference.cue | 2 +- .../reference/administration/management.cue | 2 +- website/cue/reference/cli.cue | 28 +++++++++---------- website/cue/reference/configuration.cue | 2 +- website/layouts/partials/blog/authors.html | 2 +- .../partials/components/example-configs.html | 4 +-- website/layouts/partials/data.html | 4 +-- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/website/content/en/docs/about/under-the-hood/architecture/pipeline-model.md b/website/content/en/docs/about/under-the-hood/architecture/pipeline-model.md index ee799c687e47d..bcd2144ab2aeb 100644 --- a/website/content/en/docs/about/under-the-hood/architecture/pipeline-model.md +++ b/website/content/en/docs/about/under-the-hood/architecture/pipeline-model.md @@ -10,7 +10,7 @@ Vector's pipeline model is based on a [directed acyclic graph][dag] of [componen ## Defining pipelines -A Vector pipeline is defined through a TOML, YAML, or JSON [configuration] file. For maintainability, many Vector users use configuration and data templating languages like [Jsonnet] or [CUE]. +A Vector pipeline is defined through a YAML, TOML, or JSON [configuration] file. For maintainability, many Vector users use configuration and data templating languages like [Jsonnet] or [CUE]. Configuration is checked at pipeline compile time (when Vector boots). This prevents simple mistakes and enforces DAG properties. diff --git a/website/content/en/docs/reference/configuration/_index.md b/website/content/en/docs/reference/configuration/_index.md index c9e47556afe25..699fc2a1c6a2a 100644 --- a/website/content/en/docs/reference/configuration/_index.md +++ b/website/content/en/docs/reference/configuration/_index.md @@ -270,7 +270,7 @@ environment variable example. ### Formats -Vector supports [TOML], [YAML], and [JSON] to ensure that Vector fits into your +Vector supports [YAML], [TOML], and [JSON] to ensure that Vector fits into your workflow. A side benefit of supporting YAML and JSON is that they enable you to use data templating languages such as [ytt], [Jsonnet] and [Cue]. diff --git a/website/cue/reference.cue b/website/cue/reference.cue index 17f587f64f3c4..8ad1381afdce8 100644 --- a/website/cue/reference.cue +++ b/website/cue/reference.cue @@ -455,8 +455,8 @@ _values: { Config format | Example :-------------|:------- - [TOML](\(urls.toml)) | `condition = ".status == 200"` [YAML](\(urls.yaml)) | `condition: .status == 200` + [TOML](\(urls.toml)) | `condition = ".status == 200"` [JSON](\(urls.json)) | `"condition": ".status == 200"` """ } diff --git a/website/cue/reference/administration/management.cue b/website/cue/reference/administration/management.cue index 38128dae5d264..d717b2b3ed66d 100644 --- a/website/cue/reference/administration/management.cue +++ b/website/cue/reference/administration/management.cue @@ -16,7 +16,7 @@ administration: management: { variables: { variants?: [string, ...string] - config_formats: ["toml", "yaml", "json"] + config_formats: ["yaml", "toml", "json"] } manage?: { diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index ec32672cf73df..eb9072083449d 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -136,6 +136,11 @@ cli: { type: "string" env_var: "VECTOR_CONFIG_DIR" } + "config-yaml": { + description: env_vars.VECTOR_CONFIG_YAML.description + type: "string" + env_var: "VECTOR_CONFIG_YAML" + } "config-toml": { description: env_vars.VECTOR_CONFIG_TOML.description type: "string" @@ -146,11 +151,6 @@ cli: { type: "string" env_var: "VECTOR_CONFIG_JSON" } - "config-yaml": { - description: env_vars.VECTOR_CONFIG_YAML.description - type: "string" - env_var: "VECTOR_CONFIG_YAML" - } "graceful-shutdown-limit-secs": { description: env_vars.VECTOR_GRACEFUL_SHUTDOWN_LIMIT_SECS.description default: env_vars.VECTOR_GRACEFUL_SHUTDOWN_LIMIT_SECS.type.uint.default @@ -200,7 +200,7 @@ cli: { You can also visualize the output online at [webgraphviz.com](http://www.webgraphviz.com/). """ - example: "vector graph --config /etc/vector/vector.toml | dot -Tsvg > graph.svg" + example: "vector graph --config /etc/vector/vector.yaml | dot -Tsvg > graph.svg" options: _core_options } @@ -333,8 +333,8 @@ cli: { type: "enum" default: "json" enum: { - json: "Output events as JSON" yaml: "Output events as YAML" + json: "Output events as JSON" logfmt: "Output events as logfmt" } } @@ -416,24 +416,24 @@ cli: { } options: { - "config-toml": { + "config-yaml": { description: """ Any number of Vector config files to validate. - TOML file format is assumed. + YAML file format is assumed. """ type: "string" } - "config-json": { + "config-toml": { description: """ Any number of Vector config files to validate. - JSON file format is assumed. + TOML file format is assumed. """ type: "string" } - "config-yaml": { + "config-json": { description: """ Any number of Vector config files to validate. - YAML file format is assumed. + JSON file format is assumed. """ type: "string" } @@ -537,7 +537,7 @@ cli: { Read configuration from one or more files. Wildcard paths are supported. If no files are specified the default config path `/etc/vector/vector.toml` is targeted. TOML, YAML and JSON file formats are supported. The format to interpret the file with is determined from - the file extension (`.toml`, `.yaml`, `.json`). Vector falls back to TOML if it can't + the file extension (`.yaml`, `.toml`, `.json`). Vector falls back to YAML if it can't detect a supported format. """ type: string: { diff --git a/website/cue/reference/configuration.cue b/website/cue/reference/configuration.cue index bf1a7e8f98453..cb501633edb63 100644 --- a/website/cue/reference/configuration.cue +++ b/website/cue/reference/configuration.cue @@ -701,7 +701,7 @@ configuration: { formats: { title: "Formats" body: """ - Vector supports [TOML](\(urls.toml)), [YAML](\(urls.yaml)), and [JSON](\(urls.json)) to + Vector supports [YAML](\(urls.yaml)), [TOML](\(urls.toml)), and [JSON](\(urls.json)) to ensure Vector fits into your workflow. A side benefit of supporting YAML and JSON is that they enable you to use data templating languages such as [ytt](\(urls.ytt)), [Jsonnet](\(urls.jsonnet)) and [Cue](\(urls.cue)). diff --git a/website/layouts/partials/blog/authors.html b/website/layouts/partials/blog/authors.html index 952f5335b1c18..47e822ed6754a 100644 --- a/website/layouts/partials/blog/authors.html +++ b/website/layouts/partials/blog/authors.html @@ -19,4 +19,4 @@ {{ end }} - \ No newline at end of file + diff --git a/website/layouts/partials/components/example-configs.html b/website/layouts/partials/components/example-configs.html index 17fe753299052..79a75c81bbe48 100644 --- a/website/layouts/partials/components/example-configs.html +++ b/website/layouts/partials/components/example-configs.html @@ -1,6 +1,6 @@ {{ $examples := .examples }} {{ $levels := slice "common" "advanced" }} -{{ $formats := slice "toml" "yaml" "json" }} +{{ $formats := slice "yaml" "toml" "json" }}
{{ partial "heading.html" (dict "text" "Example configurations" "level" 3) }} @@ -34,4 +34,4 @@ {{ end }}
{{ end }} - \ No newline at end of file + diff --git a/website/layouts/partials/data.html b/website/layouts/partials/data.html index 55f14ab1c44a5..a4d3fa1e3dc6e 100644 --- a/website/layouts/partials/data.html +++ b/website/layouts/partials/data.html @@ -318,7 +318,7 @@ {{/* Source/transform/sink examples */}} {{ with .component_examples }} - {{ $formats := slice "toml" "yaml" "json" }} + {{ $formats := slice "yaml" "toml" "json" }} {{ $defaultFormat := index $formats 0 }}
{{ range . }} @@ -2193,7 +2193,7 @@

{{ partial "heading.html" (dict "text" .title "level" 4) }} - {{ $formats := slice "toml" "yaml" "json" }} + {{ $formats := slice "yaml" "toml" "json" }}
{{ range $formats }}