From d13002a92d0d216caee9d99114f06025ad21c955 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Thu, 11 Oct 2018 20:31:44 +0300 Subject: [PATCH 1/2] Fixed slow pipe reading on windows It looks like the OS has the internal buffer around 4KB and with any buffer over this size `read_pipe` will end the reading loop after first try despite that reading a pipe may pump a new data and it can be read immediately. --- src/engine/execnt.c | 64 ++++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/src/engine/execnt.c b/src/engine/execnt.c index 431d11122..4e512f149 100644 --- a/src/engine/execnt.c +++ b/src/engine/execnt.c @@ -767,55 +767,35 @@ static void read_pipe DWORD bytesInBuffer = 0; DWORD bytesAvailable = 0; - do + for (;;) { /* check if we have any data to read */ if ( !PeekNamedPipe( in, ioBuffer, IO_BUFFER_SIZE, &bytesInBuffer, - &bytesAvailable, NULL ) ) - bytesAvailable = 0; + &bytesAvailable, NULL ) || bytesAvailable == 0 ) + return; - /* read in the available data */ - if ( bytesAvailable > 0 ) + /* we only read in the available bytes, to avoid blocking */ + if ( !ReadFile( in, ioBuffer, bytesAvailable <= IO_BUFFER_SIZE ? + bytesAvailable : IO_BUFFER_SIZE, &bytesInBuffer, NULL ) || bytesInBuffer == 0 ) + return; + + /* Clean up some illegal chars. */ + int i; + for ( i = 0; i < bytesInBuffer; ++i ) { - /* we only read in the available bytes, to avoid blocking */ - if ( ReadFile( in, ioBuffer, bytesAvailable <= IO_BUFFER_SIZE ? - bytesAvailable : IO_BUFFER_SIZE, &bytesInBuffer, NULL ) ) - { - if ( bytesInBuffer > 0 ) - { - /* Clean up some illegal chars. */ - int i; - for ( i = 0; i < bytesInBuffer; ++i ) - { - if ( ( (unsigned char)ioBuffer[ i ] < 1 ) ) - ioBuffer[ i ] = '?'; - } - /* Null, terminate. */ - ioBuffer[ bytesInBuffer ] = '\0'; - /* Append to the output. */ - string_append( out, ioBuffer ); - /* Copy it to our output if appropriate */ - if ( forwarding_mode == FORWARD_PIPE_STDOUT ) - out_data( ioBuffer ); - else if ( forwarding_mode == FORWARD_PIPE_STDERR ) - err_data( ioBuffer ); - /* Subtract what we read in. */ - bytesAvailable -= bytesInBuffer; - } - else - { - /* Likely read a error, bail out. */ - bytesAvailable = 0; - } - } - else - { - /* Definitely read a error, bail out. */ - bytesAvailable = 0; - } + if ( ( (unsigned char)ioBuffer[ i ] < 1 ) ) + ioBuffer[ i ] = '?'; } + /* Null, terminate. */ + ioBuffer[ bytesInBuffer ] = '\0'; + /* Append to the output. */ + string_append( out, ioBuffer ); + /* Copy it to our output if appropriate */ + if ( forwarding_mode == FORWARD_PIPE_STDOUT ) + out_data( ioBuffer ); + else if ( forwarding_mode == FORWARD_PIPE_STDERR ) + err_data( ioBuffer ); } - while ( bytesAvailable > 0 ); } #define EARLY_OUTPUT( cmd ) \ From e7e55d0cc950f5d7645c9b59ada9632d83e94f65 Mon Sep 17 00:00:00 2001 From: Nikita Kniazev Date: Thu, 11 Oct 2018 23:21:23 +0300 Subject: [PATCH 2/2] Do not read pipe content at available size peeking Currently the same data is read twice. The first time a buffer is filled by `PeekNamedPipe` and second time it is overwritten with the same content by `ReadFile`. --- src/engine/execnt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/execnt.c b/src/engine/execnt.c index 4e512f149..a712e15d3 100644 --- a/src/engine/execnt.c +++ b/src/engine/execnt.c @@ -770,7 +770,7 @@ static void read_pipe for (;;) { /* check if we have any data to read */ - if ( !PeekNamedPipe( in, ioBuffer, IO_BUFFER_SIZE, &bytesInBuffer, + if ( !PeekNamedPipe( in, NULL, IO_BUFFER_SIZE, NULL, &bytesAvailable, NULL ) || bytesAvailable == 0 ) return;