diff --git a/tests/clickhouse-test-runner/README.md b/tests/clickhouse-test-runner/README.md index 5b81d2e8..2aed73e5 100644 --- a/tests/clickhouse-test-runner/README.md +++ b/tests/clickhouse-test-runner/README.md @@ -86,8 +86,8 @@ The script builds the program with [`scripts/build.sh`](scripts/build.sh) (CMake, in `.build`, with `-DBUILD_CLICKHOUSE_TEST_RUNNER=ON -DCH_USE_3X_API=ON`), links it as `.bin/clickhouse` and `.bin/clickhouse-client`, puts `.bin` first in `PATH` for the runner only, and starts `tests/clickhouse-test`. It gives -the runner `--no-stateful --no-random-settings --no-random-merge-tree-settings`; -other arguments go to the runner after these. `UPSTREAM_TEST_LIST` selects +the runner `--no-stateful --no-random-settings --no-random-merge-tree-settings +--dont_retry_failures`; other arguments go to the runner after these. `UPSTREAM_TEST_LIST` selects other tests, also tests that are not in the allowlist. For example, to run one test: @@ -262,7 +262,8 @@ some tests read their own queries from `system.query_log`. to the server as it is, with its `FORMAT` clause: over the native protocol the server ignores the clause and sends `Native` blocks. The names and types of `...WithNames[AndTypes]` are `Block::GetColumnName` and - `Type::GetName` of the client. After `SET dialect` to another language + `Type::GetName` of the client; a Raw format writes them without escapes, + as its values. After `SET dialect` to another language (KQL, PRQL, ...), the program does not look for a `FORMAT` clause. - **Values.** The rendering follows the column type (`Column::Type()`). Strings are escaped at the top level and quoted inside `Array`, `Tuple` @@ -342,6 +343,8 @@ change this directory, the workflow, or the library (`clickhouse/`, `CLICKHOUSE_SKIP_USER_SETUP=1` and the host name of the runner (see [Run the tests locally](#run-the-tests-locally)). Shard 1 also runs the live tests of the harness. +- A failed test fails the job: the script turns off the runner's own + retries (`--dont_retry_failures`), and CI does not set `RETRY_FAILED`. - The diffs of the failed tests are in the job log. A failed job also keeps the `.stdout` and `.stderr` files of its failed tests, the log of the program (`CLICKHOUSE_CLIENT_CLI_LOG`) and the error log of the server as diff --git a/tests/clickhouse-test-runner/runner.cpp b/tests/clickhouse-test-runner/runner.cpp index f62e3dd3..eea2f027 100644 --- a/tests/clickhouse-test-runner/runner.cpp +++ b/tests/clickhouse-test-runner/runner.cpp @@ -202,12 +202,18 @@ void Runner::WriteBlock(const clickhouse::Block& block, const OutputFormat& form const size_t columns = block.GetColumnCount(); if (!state.header_written) { state.header_written = true; - auto header = [&](auto get) { + // A Raw format writes the header without escapes, as its rows + // (TabSeparatedRowOutputFormat::writeLine on the server). + auto header = [&](auto get) { for (size_t i = 0; i < columns; ++i) { if (i > 0) { result_ += '\t'; } - WriteEscapedString(result_, get(i), false); + if (format.raw) { + result_ += get(i); + } else { + WriteEscapedString(result_, get(i), false); + } } result_ += '\n'; }; diff --git a/tests/clickhouse-test-runner/scripts/run-upstream-tests.sh b/tests/clickhouse-test-runner/scripts/run-upstream-tests.sh index 90a38ce5..494c45ac 100755 --- a/tests/clickhouse-test-runner/scripts/run-upstream-tests.sh +++ b/tests/clickhouse-test-runner/scripts/run-upstream-tests.sh @@ -122,7 +122,11 @@ source "${SCRIPT_DIR}/selectors.sh" build_selectors "${tests[@]}" cd "${UPSTREAM_CLICKHOUSE_DIR}" -runner=(python3 tests/clickhouse-test --no-stateful --no-random-settings --no-random-merge-tree-settings) +# --dont_retry_failures: by default the runner runs a failed test again when +# its output has one of its MESSAGES_TO_RETRY (for example "No such file or +# directory"), and reports a pass as OK. RETRY_FAILED is the only retry here. +runner=(python3 tests/clickhouse-test --no-stateful --no-random-settings --no-random-merge-tree-settings + --dont_retry_failures) retries="${RETRY_FAILED:-0}" if ! [[ "${retries}" =~ ^[0-9]+$ ]]; then diff --git a/tests/clickhouse-test-runner/sql_text.cpp b/tests/clickhouse-test-runner/sql_text.cpp index 4659ab16..f57be100 100644 --- a/tests/clickhouse-test-runner/sql_text.cpp +++ b/tests/clickhouse-test-runner/sql_text.cpp @@ -132,6 +132,9 @@ std::vector TopLevelWords(std::string_view sql) { i = SkipLineComment(sql, i); } else if (StartsWithAt(sql, i, "/*")) { i = SkipBlockComment(sql, i); + } else if (c == '$') { + const size_t end = HeredocEnd(sql, i); + i = end > 0 ? end : i + 1; } else if (c == '(' || c == '[') { ++depth; ++i; diff --git a/tests/clickhouse-test-runner/sql_text.h b/tests/clickhouse-test-runner/sql_text.h index 9ba697af..131db98e 100644 --- a/tests/clickhouse-test-runner/sql_text.h +++ b/tests/clickhouse-test-runner/sql_text.h @@ -39,7 +39,7 @@ bool HasSignificantToken(std::string_view text); /// and opening parentheses. std::string LeadingKeyword(std::string_view sql); -/// The words of `sql` outside quotes, comments and parentheses. +/// The words of `sql` outside quotes, heredocs, comments and parentheses. std::vector TopLevelWords(std::string_view sql); /// SQL words that can follow a column or an alias named `format`. No input diff --git a/tests/clickhouse-test-runner/ut/live_ut.cpp b/tests/clickhouse-test-runner/ut/live_ut.cpp index 435dd48d..615b7be7 100644 --- a/tests/clickhouse-test-runner/ut/live_ut.cpp +++ b/tests/clickhouse-test-runner/ut/live_ut.cpp @@ -175,6 +175,25 @@ TEST(Live, ValuesAndLineAsStringMatchServer) { } } +TEST(Live, HeaderOfFormatMatchesServer) { + REQUIRE_LIVE(); + // The header of a Raw format has no escapes, as its rows: the names, and + // the types (the Enum type name has quotes and a backslash). Contrast: + // TSVWithNames and TSVWithNamesAndTypes escape the header. + const std::string names = "SELECT 1 AS `a\\tb\\\\c'd`, 'x' AS `\xC3\xA9\\nf` FORMAT "; + const std::string types = "SELECT CAST('a' AS Enum8('a\\tb' = 1, 'a' = 2)) AS e FORMAT "; + const std::vector queries = { + names + "TSVWithNames", names + "TSVRawWithNames", names + "TabSeparatedRawWithNamesAndTypes", + types + "TSVWithNamesAndTypes", types + "TSVRawWithNamesAndTypes", + }; + for (const auto& query : queries) { + const auto want = ServerTSV(query); + const auto got = RunShim(query); + EXPECT_EQ(0, got.code) << query << "\n" << got.err; + EXPECT_EQ(want, got.out) << query; + } +} + TEST(Live, SessionState) { REQUIRE_LIVE(); // SET, temporary tables and inline INSERT data last for the whole diff --git a/tests/clickhouse-test-runner/ut/split_ut.cpp b/tests/clickhouse-test-runner/ut/split_ut.cpp index a9b48ff2..c945f2a1 100644 --- a/tests/clickhouse-test-runner/ut/split_ut.cpp +++ b/tests/clickhouse-test-runner/ut/split_ut.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include "split.h" @@ -61,6 +62,9 @@ TEST(SplitStatements, Cases) { {"insert select with an implicit alias after format", "INSERT INTO t SELECT format x FROM s;\nSELECT 1;", {"INSERT INTO t SELECT format x FROM s;", "SELECT 1;"}}, + {"insert select with a heredoc before a format alias", + "INSERT INTO t SELECT $$ FROM $$ AS x, format y FROM s;\nSELECT 1;", + {"INSERT INTO t SELECT $$ FROM $$ AS x, format y FROM s;", "SELECT 1;"}}, {"insert select with format DIV", "INSERT INTO t SELECT format DIV 2 FROM s;\nSELECT 1;", {"INSERT INTO t SELECT format DIV 2 FROM s;", "SELECT 1;"}}, @@ -136,6 +140,22 @@ TEST(SqlText, LeadingKeyword) { EXPECT_EQ("", LeadingKeyword("-- only a comment")); } +TEST(SqlText, TopLevelWordsSkipsLiteralsAndComments) { + const std::vector> cases = { + {"SELECT a, 'b c', `d`, \"e\", $$f g$$, $t$h$t$ -- i\nFROM (SELECT j) /* k */", "SELECT a FROM"}, + {"INSERT INTO t SELECT $$x FROM y$$ FROM (SELECT z)", "INSERT INTO t SELECT FROM"}, + // A `$` that does not start a heredoc is not skipped with the text after it. + {"SELECT $a$b$ FROM t", "SELECT a b FROM t"}, + }; + for (const auto& [sql, want] : cases) { + std::string got; + for (const auto w : TopLevelWords(sql)) { + got += (got.empty() ? "" : " ") + std::string(w); + } + EXPECT_EQ(want, got) << sql; + } +} + TEST(SqlText, TestTagsLength) { EXPECT_EQ(0u, TestTagsLength("SELECT 1")); EXPECT_EQ(0u, TestTagsLength("-- a comment\nSELECT 1"));