1
0
mirror of https://github.com/git/git synced 2024-07-05 00:58:49 +00:00
git/t/helper/test-pkt-line.c

189 lines
4.5 KiB
C
Raw Permalink Normal View History

#include "git-compat-util.h"
#include "test-tool.h"
#include "pkt-line.h"
#include "sideband.h"
#include "write-or-die.h"
test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-12-17 14:41:36 +00:00
#include "parse-options.h"
static void pack_line(const char *line)
{
if (!strcmp(line, "0000") || !strcmp(line, "0000\n"))
packet_flush(1);
else if (!strcmp(line, "0001") || !strcmp(line, "0001\n"))
packet_delim(1);
else
packet_write_fmt(1, "%s", line);
}
static void pack(int argc, const char **argv)
{
if (argc) { /* read from argv */
int i;
for (i = 0; i < argc; i++)
pack_line(argv[i]);
} else { /* read from stdin */
char line[LARGE_PACKET_MAX];
while (fgets(line, sizeof(line), stdin)) {
pack_line(line);
}
}
}
static void pack_raw_stdin(void)
{
struct strbuf sb = STRBUF_INIT;
if (strbuf_read(&sb, 0, 0) < 0)
die_errno("failed to read from stdin");
packet_write(1, sb.buf, sb.len);
strbuf_release(&sb);
}
static void unpack(void)
{
struct packet_reader reader;
packet_reader_init(&reader, 0, NULL, 0,
PACKET_READ_GENTLE_ON_EOF |
PACKET_READ_CHOMP_NEWLINE);
while (packet_reader_read(&reader) != PACKET_READ_EOF) {
switch (reader.status) {
case PACKET_READ_EOF:
break;
case PACKET_READ_NORMAL:
printf("%s\n", reader.line);
break;
case PACKET_READ_FLUSH:
printf("0000\n");
break;
case PACKET_READ_DELIM:
printf("0001\n");
break;
case PACKET_READ_RESPONSE_END:
printf("0002\n");
break;
}
}
}
test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-12-17 14:41:36 +00:00
static void unpack_sideband(int argc, const char **argv)
{
struct packet_reader reader;
test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-12-17 14:41:36 +00:00
int options = PACKET_READ_GENTLE_ON_EOF;
int chomp_newline = 1;
int reader_use_sideband = 0;
const char *const unpack_sideband_usage[] = {
"test_tool unpack_sideband [options...]", NULL
};
struct option cmd_options[] = {
OPT_BOOL(0, "reader-use-sideband", &reader_use_sideband,
"set use_sideband bit for packet reader (Default: off)"),
OPT_BOOL(0, "chomp-newline", &chomp_newline,
"chomp newline in packet (Default: on)"),
OPT_END()
};
argc = parse_options(argc, argv, "", cmd_options, unpack_sideband_usage,
0);
if (argc > 0)
usage_msg_opt(_("too many arguments"), unpack_sideband_usage,
cmd_options);
if (chomp_newline)
options |= PACKET_READ_CHOMP_NEWLINE;
packet_reader_init(&reader, 0, NULL, 0, options);
reader.use_sideband = reader_use_sideband;
while (packet_reader_read(&reader) != PACKET_READ_EOF) {
int band;
int fd;
switch (reader.status) {
case PACKET_READ_EOF:
break;
case PACKET_READ_NORMAL:
test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-12-17 14:41:36 +00:00
/*
* When the "use_sideband" field of the reader is turned
* on, sideband packets other than the payload have been
* parsed and consumed in packet_reader_read(), and only
* the payload arrives here.
*/
if (reader.use_sideband) {
write_or_die(1, reader.line, reader.pktlen - 1);
break;
}
band = reader.line[0] & 0xff;
if (band < 1 || band > 2)
continue; /* skip non-sideband packets */
fd = band;
write_or_die(fd, reader.line + 1, reader.pktlen - 1);
break;
case PACKET_READ_FLUSH:
return;
case PACKET_READ_DELIM:
case PACKET_READ_RESPONSE_END:
break;
}
}
}
sideband: avoid reporting incomplete sideband messages In 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) we tried to ensure that the "Total 3" message could be grepped in Git's output, even if it sometimes got chopped up into multiple lines in the trace machinery. However, the first instance where this mattered now goes through the sideband machinery, where it is _still_ possible for messages to get chopped up: it *is* possible for the standard error stream to be sent byte-for-byte and hence it can be easily interrupted. Meaning: it is possible for the single line that we're looking for to be chopped up into multiple sideband packets, with a primary packet being delivered between them. This seems to happen occasionally in the `vs-test` part of our CI builds, i.e. with binaries built using Visual C, but not when building with GCC or clang; The symptom is that t5500.43 fails to find a line matching `remote: Total 3` in the `log` file, which ends in something along these lines: remote: Tota remote: l 3 (delta 0), reused 0 (delta 0), pack-reused 0 This should not happen, though: we have code in `demultiplex_sideband()` _specifically_ to stitch back together lines that were delivered in separate sideband packets. However, this stitching was broken in a subtle way in fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16): before that change, incomplete sideband lines would not be flushed upon receiving a primary packet, but after that patch, they would be. The subtleness of this bug comes from the fact that it is easy to get confused by the ambiguous meaning of the `break` keyword: after writing the primary packet contents, the `break;` in the original version of `recv_sideband()` does _not_ break out of the `while` loop, but instead only ends the `switch` case: while (!retval) { [...] switch (band) { [...] case 1: /* Write the contents of the primary packet */ write_or_die(out, buf + 1, len); /* Here, we do *not* break out of the loop, `retval` is unchanged */ break; [...] } if (outbuf.len) { /* Write any remaining sideband messages lacking a trailing LF */ strbuf_addch(&outbuf, '\n'); xwrite(2, outbuf.buf, outbuf.len); } In contrast, after fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16), the body of the `while` loop was extracted into `demultiplex_sideband()`, crucially _including_ the logic to write incomplete sideband messages: switch (band) { [...] case 1: *sideband_type = SIDEBAND_PRIMARY; /* This does not break out of the loop: the loop is in the caller */ break; [...] } cleanup: [...] /* This logic is now no longer _outside_ the loop but _inside_ */ if (scratch->len) { strbuf_addch(scratch, '\n'); xwrite(2, scratch->buf, scratch->len); } The correct way to fix this is to return from `demultiplex_sideband()` early. The caller will then write out the contents of the primary packet and continue looping. The `scratch` buffer for incomplete sideband messages is owned by that caller, and will continue to accumulate the remainder(s) of those messages. The loop will only end once `demultiplex_sideband()` returned non-zero _and_ did not indicate a primary packet, which is the case only when we hit the `cleanup:` path, in which we take care of flushing any unfinished sideband messages and release the `scratch` buffer. To ensure that this does not get broken again, we introduce a pair of subcommands of the `pkt-line` test helper that specifically chop up the sideband message and squeeze a primary packet into the middle. Final note: The other test case touched by 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) is not affected by this issue because the sideband machinery is not involved there. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-10-19 19:35:40 +00:00
static int send_split_sideband(void)
{
test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-12-17 14:41:36 +00:00
const char *foo = "Foo.\n";
const char *bar = "Bar.\n";
sideband: avoid reporting incomplete sideband messages In 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) we tried to ensure that the "Total 3" message could be grepped in Git's output, even if it sometimes got chopped up into multiple lines in the trace machinery. However, the first instance where this mattered now goes through the sideband machinery, where it is _still_ possible for messages to get chopped up: it *is* possible for the standard error stream to be sent byte-for-byte and hence it can be easily interrupted. Meaning: it is possible for the single line that we're looking for to be chopped up into multiple sideband packets, with a primary packet being delivered between them. This seems to happen occasionally in the `vs-test` part of our CI builds, i.e. with binaries built using Visual C, but not when building with GCC or clang; The symptom is that t5500.43 fails to find a line matching `remote: Total 3` in the `log` file, which ends in something along these lines: remote: Tota remote: l 3 (delta 0), reused 0 (delta 0), pack-reused 0 This should not happen, though: we have code in `demultiplex_sideband()` _specifically_ to stitch back together lines that were delivered in separate sideband packets. However, this stitching was broken in a subtle way in fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16): before that change, incomplete sideband lines would not be flushed upon receiving a primary packet, but after that patch, they would be. The subtleness of this bug comes from the fact that it is easy to get confused by the ambiguous meaning of the `break` keyword: after writing the primary packet contents, the `break;` in the original version of `recv_sideband()` does _not_ break out of the `while` loop, but instead only ends the `switch` case: while (!retval) { [...] switch (band) { [...] case 1: /* Write the contents of the primary packet */ write_or_die(out, buf + 1, len); /* Here, we do *not* break out of the loop, `retval` is unchanged */ break; [...] } if (outbuf.len) { /* Write any remaining sideband messages lacking a trailing LF */ strbuf_addch(&outbuf, '\n'); xwrite(2, outbuf.buf, outbuf.len); } In contrast, after fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16), the body of the `while` loop was extracted into `demultiplex_sideband()`, crucially _including_ the logic to write incomplete sideband messages: switch (band) { [...] case 1: *sideband_type = SIDEBAND_PRIMARY; /* This does not break out of the loop: the loop is in the caller */ break; [...] } cleanup: [...] /* This logic is now no longer _outside_ the loop but _inside_ */ if (scratch->len) { strbuf_addch(scratch, '\n'); xwrite(2, scratch->buf, scratch->len); } The correct way to fix this is to return from `demultiplex_sideband()` early. The caller will then write out the contents of the primary packet and continue looping. The `scratch` buffer for incomplete sideband messages is owned by that caller, and will continue to accumulate the remainder(s) of those messages. The loop will only end once `demultiplex_sideband()` returned non-zero _and_ did not indicate a primary packet, which is the case only when we hit the `cleanup:` path, in which we take care of flushing any unfinished sideband messages and release the `scratch` buffer. To ensure that this does not get broken again, we introduce a pair of subcommands of the `pkt-line` test helper that specifically chop up the sideband message and squeeze a primary packet into the middle. Final note: The other test case touched by 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) is not affected by this issue because the sideband machinery is not involved there. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-10-19 19:35:40 +00:00
const char *part1 = "Hello,";
const char *primary = "\001primary: regular output\n";
const char *part2 = " world!\n";
test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-12-17 14:41:36 +00:00
/* Each sideband message has a trailing newline character. */
send_sideband(1, 2, foo, strlen(foo), LARGE_PACKET_MAX);
send_sideband(1, 2, bar, strlen(bar), LARGE_PACKET_MAX);
/*
* One sideband message is divided into part1 and part2
* by the primary message.
*/
sideband: avoid reporting incomplete sideband messages In 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) we tried to ensure that the "Total 3" message could be grepped in Git's output, even if it sometimes got chopped up into multiple lines in the trace machinery. However, the first instance where this mattered now goes through the sideband machinery, where it is _still_ possible for messages to get chopped up: it *is* possible for the standard error stream to be sent byte-for-byte and hence it can be easily interrupted. Meaning: it is possible for the single line that we're looking for to be chopped up into multiple sideband packets, with a primary packet being delivered between them. This seems to happen occasionally in the `vs-test` part of our CI builds, i.e. with binaries built using Visual C, but not when building with GCC or clang; The symptom is that t5500.43 fails to find a line matching `remote: Total 3` in the `log` file, which ends in something along these lines: remote: Tota remote: l 3 (delta 0), reused 0 (delta 0), pack-reused 0 This should not happen, though: we have code in `demultiplex_sideband()` _specifically_ to stitch back together lines that were delivered in separate sideband packets. However, this stitching was broken in a subtle way in fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16): before that change, incomplete sideband lines would not be flushed upon receiving a primary packet, but after that patch, they would be. The subtleness of this bug comes from the fact that it is easy to get confused by the ambiguous meaning of the `break` keyword: after writing the primary packet contents, the `break;` in the original version of `recv_sideband()` does _not_ break out of the `while` loop, but instead only ends the `switch` case: while (!retval) { [...] switch (band) { [...] case 1: /* Write the contents of the primary packet */ write_or_die(out, buf + 1, len); /* Here, we do *not* break out of the loop, `retval` is unchanged */ break; [...] } if (outbuf.len) { /* Write any remaining sideband messages lacking a trailing LF */ strbuf_addch(&outbuf, '\n'); xwrite(2, outbuf.buf, outbuf.len); } In contrast, after fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16), the body of the `while` loop was extracted into `demultiplex_sideband()`, crucially _including_ the logic to write incomplete sideband messages: switch (band) { [...] case 1: *sideband_type = SIDEBAND_PRIMARY; /* This does not break out of the loop: the loop is in the caller */ break; [...] } cleanup: [...] /* This logic is now no longer _outside_ the loop but _inside_ */ if (scratch->len) { strbuf_addch(scratch, '\n'); xwrite(2, scratch->buf, scratch->len); } The correct way to fix this is to return from `demultiplex_sideband()` early. The caller will then write out the contents of the primary packet and continue looping. The `scratch` buffer for incomplete sideband messages is owned by that caller, and will continue to accumulate the remainder(s) of those messages. The loop will only end once `demultiplex_sideband()` returned non-zero _and_ did not indicate a primary packet, which is the case only when we hit the `cleanup:` path, in which we take care of flushing any unfinished sideband messages and release the `scratch` buffer. To ensure that this does not get broken again, we introduce a pair of subcommands of the `pkt-line` test helper that specifically chop up the sideband message and squeeze a primary packet into the middle. Final note: The other test case touched by 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) is not affected by this issue because the sideband machinery is not involved there. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-10-19 19:35:40 +00:00
send_sideband(1, 2, part1, strlen(part1), LARGE_PACKET_MAX);
packet_write(1, primary, strlen(primary));
send_sideband(1, 2, part2, strlen(part2), LARGE_PACKET_MAX);
packet_response_end(1);
test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-12-17 14:41:36 +00:00
/*
* We use unpack_sideband() to consume packets. A flush packet
* is required to end parsing.
*/
packet_flush(1);
sideband: avoid reporting incomplete sideband messages In 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) we tried to ensure that the "Total 3" message could be grepped in Git's output, even if it sometimes got chopped up into multiple lines in the trace machinery. However, the first instance where this mattered now goes through the sideband machinery, where it is _still_ possible for messages to get chopped up: it *is* possible for the standard error stream to be sent byte-for-byte and hence it can be easily interrupted. Meaning: it is possible for the single line that we're looking for to be chopped up into multiple sideband packets, with a primary packet being delivered between them. This seems to happen occasionally in the `vs-test` part of our CI builds, i.e. with binaries built using Visual C, but not when building with GCC or clang; The symptom is that t5500.43 fails to find a line matching `remote: Total 3` in the `log` file, which ends in something along these lines: remote: Tota remote: l 3 (delta 0), reused 0 (delta 0), pack-reused 0 This should not happen, though: we have code in `demultiplex_sideband()` _specifically_ to stitch back together lines that were delivered in separate sideband packets. However, this stitching was broken in a subtle way in fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16): before that change, incomplete sideband lines would not be flushed upon receiving a primary packet, but after that patch, they would be. The subtleness of this bug comes from the fact that it is easy to get confused by the ambiguous meaning of the `break` keyword: after writing the primary packet contents, the `break;` in the original version of `recv_sideband()` does _not_ break out of the `while` loop, but instead only ends the `switch` case: while (!retval) { [...] switch (band) { [...] case 1: /* Write the contents of the primary packet */ write_or_die(out, buf + 1, len); /* Here, we do *not* break out of the loop, `retval` is unchanged */ break; [...] } if (outbuf.len) { /* Write any remaining sideband messages lacking a trailing LF */ strbuf_addch(&outbuf, '\n'); xwrite(2, outbuf.buf, outbuf.len); } In contrast, after fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16), the body of the `while` loop was extracted into `demultiplex_sideband()`, crucially _including_ the logic to write incomplete sideband messages: switch (band) { [...] case 1: *sideband_type = SIDEBAND_PRIMARY; /* This does not break out of the loop: the loop is in the caller */ break; [...] } cleanup: [...] /* This logic is now no longer _outside_ the loop but _inside_ */ if (scratch->len) { strbuf_addch(scratch, '\n'); xwrite(2, scratch->buf, scratch->len); } The correct way to fix this is to return from `demultiplex_sideband()` early. The caller will then write out the contents of the primary packet and continue looping. The `scratch` buffer for incomplete sideband messages is owned by that caller, and will continue to accumulate the remainder(s) of those messages. The loop will only end once `demultiplex_sideband()` returned non-zero _and_ did not indicate a primary packet, which is the case only when we hit the `cleanup:` path, in which we take care of flushing any unfinished sideband messages and release the `scratch` buffer. To ensure that this does not get broken again, we introduce a pair of subcommands of the `pkt-line` test helper that specifically chop up the sideband message and squeeze a primary packet into the middle. Final note: The other test case touched by 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) is not affected by this issue because the sideband machinery is not involved there. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-10-19 19:35:40 +00:00
return 0;
}
static int receive_sideband(void)
{
return recv_sideband("sideband", 0, 1);
sideband: avoid reporting incomplete sideband messages In 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) we tried to ensure that the "Total 3" message could be grepped in Git's output, even if it sometimes got chopped up into multiple lines in the trace machinery. However, the first instance where this mattered now goes through the sideband machinery, where it is _still_ possible for messages to get chopped up: it *is* possible for the standard error stream to be sent byte-for-byte and hence it can be easily interrupted. Meaning: it is possible for the single line that we're looking for to be chopped up into multiple sideband packets, with a primary packet being delivered between them. This seems to happen occasionally in the `vs-test` part of our CI builds, i.e. with binaries built using Visual C, but not when building with GCC or clang; The symptom is that t5500.43 fails to find a line matching `remote: Total 3` in the `log` file, which ends in something along these lines: remote: Tota remote: l 3 (delta 0), reused 0 (delta 0), pack-reused 0 This should not happen, though: we have code in `demultiplex_sideband()` _specifically_ to stitch back together lines that were delivered in separate sideband packets. However, this stitching was broken in a subtle way in fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16): before that change, incomplete sideband lines would not be flushed upon receiving a primary packet, but after that patch, they would be. The subtleness of this bug comes from the fact that it is easy to get confused by the ambiguous meaning of the `break` keyword: after writing the primary packet contents, the `break;` in the original version of `recv_sideband()` does _not_ break out of the `while` loop, but instead only ends the `switch` case: while (!retval) { [...] switch (band) { [...] case 1: /* Write the contents of the primary packet */ write_or_die(out, buf + 1, len); /* Here, we do *not* break out of the loop, `retval` is unchanged */ break; [...] } if (outbuf.len) { /* Write any remaining sideband messages lacking a trailing LF */ strbuf_addch(&outbuf, '\n'); xwrite(2, outbuf.buf, outbuf.len); } In contrast, after fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16), the body of the `while` loop was extracted into `demultiplex_sideband()`, crucially _including_ the logic to write incomplete sideband messages: switch (band) { [...] case 1: *sideband_type = SIDEBAND_PRIMARY; /* This does not break out of the loop: the loop is in the caller */ break; [...] } cleanup: [...] /* This logic is now no longer _outside_ the loop but _inside_ */ if (scratch->len) { strbuf_addch(scratch, '\n'); xwrite(2, scratch->buf, scratch->len); } The correct way to fix this is to return from `demultiplex_sideband()` early. The caller will then write out the contents of the primary packet and continue looping. The `scratch` buffer for incomplete sideband messages is owned by that caller, and will continue to accumulate the remainder(s) of those messages. The loop will only end once `demultiplex_sideband()` returned non-zero _and_ did not indicate a primary packet, which is the case only when we hit the `cleanup:` path, in which we take care of flushing any unfinished sideband messages and release the `scratch` buffer. To ensure that this does not get broken again, we introduce a pair of subcommands of the `pkt-line` test helper that specifically chop up the sideband message and squeeze a primary packet into the middle. Final note: The other test case touched by 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) is not affected by this issue because the sideband machinery is not involved there. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-10-19 19:35:40 +00:00
}
int cmd__pkt_line(int argc, const char **argv)
{
if (argc < 2)
die("too few arguments");
if (!strcmp(argv[1], "pack"))
pack(argc - 2, argv + 2);
else if (!strcmp(argv[1], "pack-raw-stdin"))
pack_raw_stdin();
else if (!strcmp(argv[1], "unpack"))
unpack();
else if (!strcmp(argv[1], "unpack-sideband"))
test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-12-17 14:41:36 +00:00
unpack_sideband(argc - 1, argv + 1);
sideband: avoid reporting incomplete sideband messages In 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) we tried to ensure that the "Total 3" message could be grepped in Git's output, even if it sometimes got chopped up into multiple lines in the trace machinery. However, the first instance where this mattered now goes through the sideband machinery, where it is _still_ possible for messages to get chopped up: it *is* possible for the standard error stream to be sent byte-for-byte and hence it can be easily interrupted. Meaning: it is possible for the single line that we're looking for to be chopped up into multiple sideband packets, with a primary packet being delivered between them. This seems to happen occasionally in the `vs-test` part of our CI builds, i.e. with binaries built using Visual C, but not when building with GCC or clang; The symptom is that t5500.43 fails to find a line matching `remote: Total 3` in the `log` file, which ends in something along these lines: remote: Tota remote: l 3 (delta 0), reused 0 (delta 0), pack-reused 0 This should not happen, though: we have code in `demultiplex_sideband()` _specifically_ to stitch back together lines that were delivered in separate sideband packets. However, this stitching was broken in a subtle way in fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16): before that change, incomplete sideband lines would not be flushed upon receiving a primary packet, but after that patch, they would be. The subtleness of this bug comes from the fact that it is easy to get confused by the ambiguous meaning of the `break` keyword: after writing the primary packet contents, the `break;` in the original version of `recv_sideband()` does _not_ break out of the `while` loop, but instead only ends the `switch` case: while (!retval) { [...] switch (band) { [...] case 1: /* Write the contents of the primary packet */ write_or_die(out, buf + 1, len); /* Here, we do *not* break out of the loop, `retval` is unchanged */ break; [...] } if (outbuf.len) { /* Write any remaining sideband messages lacking a trailing LF */ strbuf_addch(&outbuf, '\n'); xwrite(2, outbuf.buf, outbuf.len); } In contrast, after fbd76cd450 (sideband: reverse its dependency on pkt-line, 2019-01-16), the body of the `while` loop was extracted into `demultiplex_sideband()`, crucially _including_ the logic to write incomplete sideband messages: switch (band) { [...] case 1: *sideband_type = SIDEBAND_PRIMARY; /* This does not break out of the loop: the loop is in the caller */ break; [...] } cleanup: [...] /* This logic is now no longer _outside_ the loop but _inside_ */ if (scratch->len) { strbuf_addch(scratch, '\n'); xwrite(2, scratch->buf, scratch->len); } The correct way to fix this is to return from `demultiplex_sideband()` early. The caller will then write out the contents of the primary packet and continue looping. The `scratch` buffer for incomplete sideband messages is owned by that caller, and will continue to accumulate the remainder(s) of those messages. The loop will only end once `demultiplex_sideband()` returned non-zero _and_ did not indicate a primary packet, which is the case only when we hit the `cleanup:` path, in which we take care of flushing any unfinished sideband messages and release the `scratch` buffer. To ensure that this does not get broken again, we introduce a pair of subcommands of the `pkt-line` test helper that specifically chop up the sideband message and squeeze a primary packet into the middle. Final note: The other test case touched by 2b695ecd74d (t5500: count objects through stderr, not trace, 2020-05-06) is not affected by this issue because the sideband machinery is not involved there. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-10-19 19:35:40 +00:00
else if (!strcmp(argv[1], "send-split-sideband"))
send_split_sideband();
else if (!strcmp(argv[1], "receive-sideband"))
receive_sideband();
else
die("invalid argument '%s'", argv[1]);
return 0;
}