Quellcode durchsuchen

Update to latest zsv.

Fixes column count issue with no EOL on last row with empty last column.
Brucey vor 2 Jahren
Ursprung
Commit
c5a938f6c4

+ 1 - 0
csv.mod/zsv/Makefile

@@ -55,6 +55,7 @@ lib:
 
 
 test:
 test:
 	@${MAKE} -C app test CONFIGFILE=${CONFIGFILEPATH}
 	@${MAKE} -C app test CONFIGFILE=${CONFIGFILEPATH}
+	@${MAKE} -C examples/lib test CONFIGFILE=${CONFIGFILEPATH}
 
 
 install:
 install:
 	@${MAKE} -C src install CONFIGFILE=${CONFIGFILEPATH}
 	@${MAKE} -C src install CONFIGFILE=${CONFIGFILEPATH}

+ 4 - 0
csv.mod/zsv/data/test/desc-trim.csv

@@ -0,0 +1,4 @@
+abc,def,ghi
+x,A,1
+x,B,1
+x, A,1

+ 4 - 0
csv.mod/zsv/data/test/no-eol-1.csv

@@ -0,0 +1,4 @@
+one,two
+1,2
+3,
+4,

+ 4 - 0
csv.mod/zsv/data/test/no-eol-2.csv

@@ -0,0 +1,4 @@
+one,two
+1,2
+3,
+4, 

+ 4 - 0
csv.mod/zsv/data/test/no-eol-3.csv

@@ -0,0 +1,4 @@
+one,two
+1,2
+3,
+4

+ 3 - 0
csv.mod/zsv/data/test/no-eol-4.csv

@@ -0,0 +1,3 @@
+one,two
+1,2
+3,

+ 21 - 0
csv.mod/zsv/examples/lib/Makefile

@@ -29,6 +29,17 @@ ifeq ($(WIN),)
   endif
   endif
 endif
 endif
 
 
+TEST_DATA_DIR=${THIS_MAKEFILE_DIR}/../../data
+TMP_DIR=/tmp
+COLOR_NONE=\033[0m
+COLOR_GREEN=\033[1;32m
+COLOR_RED=\033[1;31m
+COLOR_BLUE=\033[1;34m
+COLOR_PINK=\033[1;35m
+
+TEST_PASS=echo "${COLOR_BLUE}$@: ${COLOR_GREEN}Passed${COLOR_NONE}"
+TEST_FAIL=(echo "${COLOR_BLUE}$@: ${COLOR_RED}Failed!${COLOR_NONE}" && exit 1)
+
 EXE=
 EXE=
 ifeq ($(WIN),1)
 ifeq ($(WIN),1)
   EXE=.exe
   EXE=.exe
@@ -61,6 +72,16 @@ help:
 
 
 build: simple print_my_column parse_by_chunk pull
 build: simple print_my_column parse_by_chunk pull
 
 
+test: test-eol
+
+test-eol: test-eol-1 test-eol-2 test-eol-3 test-eol-4
+
+test-eol-%: build/simple${EXE} build/pull${EXE}
+	@$< ${TEST_DATA_DIR}/test/no-eol-$*.csv > ${TMP_DIR}/[email protected]
+	@cmp ${TMP_DIR}/[email protected] test/expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL}
+
+	@build/pull${EXE} ${TEST_DATA_DIR}/test/no-eol-$*.csv > ${TMP_DIR}/[email protected]
+	@cmp ${TMP_DIR}/[email protected] test/expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL}
 
 
 simple print_my_column parse_by_chunk pull: % : ${BUILD_DIR}/%${EXE}
 simple print_my_column parse_by_chunk pull: % : ${BUILD_DIR}/%${EXE}
 	@echo Built $<
 	@echo Built $<

+ 4 - 0
csv.mod/zsv/examples/lib/test/expected/test-eol-1.out

@@ -0,0 +1,4 @@
+Row 1 has 2 columns of which 2 are non-blank
+Row 2 has 2 columns of which 2 are non-blank
+Row 3 has 2 columns of which 1 is non-blank
+Row 4 has 2 columns of which 1 is non-blank

+ 4 - 0
csv.mod/zsv/examples/lib/test/expected/test-eol-2.out

@@ -0,0 +1,4 @@
+Row 1 has 2 columns of which 2 are non-blank
+Row 2 has 2 columns of which 2 are non-blank
+Row 3 has 2 columns of which 1 is non-blank
+Row 4 has 2 columns of which 2 are non-blank

+ 4 - 0
csv.mod/zsv/examples/lib/test/expected/test-eol-3.out

@@ -0,0 +1,4 @@
+Row 1 has 2 columns of which 2 are non-blank
+Row 2 has 2 columns of which 2 are non-blank
+Row 3 has 2 columns of which 1 is non-blank
+Row 4 has 1 columns of which 1 is non-blank

+ 3 - 0
csv.mod/zsv/examples/lib/test/expected/test-eol-4.out

@@ -0,0 +1,3 @@
+Row 1 has 2 columns of which 2 are non-blank
+Row 2 has 2 columns of which 2 are non-blank
+Row 3 has 2 columns of which 1 is non-blank

+ 1 - 1
csv.mod/zsv/src/zsv.c

@@ -347,7 +347,7 @@ enum zsv_status zsv_finish(struct zsv_scanner *scanner) {
   if(!scanner->finished) {
   if(!scanner->finished) {
     scanner->finished = 1;
     scanner->finished = 1;
     if(!scanner->abort) {
     if(!scanner->abort) {
-      if(scanner->scanned_length > scanner->cell_start)
+      if(scanner->scanned_length > 0 && scanner->scanned_length >= scanner->cell_start)
         cell_dl(scanner, scanner->buff.buff + scanner->cell_start,
         cell_dl(scanner, scanner->buff.buff + scanner->cell_start,
                 scanner->scanned_length - scanner->cell_start);
                 scanner->scanned_length - scanner->cell_start);
       if(scanner->have_cell) {
       if(scanner->have_cell) {

+ 6 - 5
csv.mod/zsv/src/zsv_internal.c

@@ -247,10 +247,7 @@ __attribute__((always_inline)) static inline void zsv_clear_cell(struct zsv_scan
 }
 }
 
 
 // always_inline has a noticeable impact. do not remove without benchmarking!
 // always_inline has a noticeable impact. do not remove without benchmarking!
-#ifdef NDEBUG
-__attribute__((always_inline)) static inline
-#endif
-void cell_dl(struct zsv_scanner * scanner, unsigned char * s, size_t n) {
+__attribute__((always_inline)) static inline void cell_dl(struct zsv_scanner * scanner, unsigned char * s, size_t n) {
   // handle quoting
   // handle quoting
   if(UNLIKELY(scanner->quoted > 0)) {
   if(UNLIKELY(scanner->quoted > 0)) {
     if(LIKELY(scanner->quote_close_position + 1 == n)) {
     if(LIKELY(scanner->quote_close_position + 1 == n)) {
@@ -375,7 +372,7 @@ static inline enum zsv_status cell_and_row_dl(struct zsv_scanner *scanner, unsig
   only for each corresponding non-zero highest-bit value in the vector)
   only for each corresponding non-zero highest-bit value in the vector)
 */
 */
 
 
-# ifdef __EMSCRIPTEN__
+# if defined(__EMSCRIPTEN__) && defined(__SIMD128__)
 
 
 #include <wasm_simd128.h>
 #include <wasm_simd128.h>
 #define movemask_pseudo(x) wasm_i8x16_bitmask(x)
 #define movemask_pseudo(x) wasm_i8x16_bitmask(x)
@@ -406,6 +403,10 @@ typedef char zsv_c_vector __attribute__ ((vector_size (VECTOR_BYTES)));
 # else
 # else
 
 
 // slow path
 // slow path
+# if defined(__EMSCRIPTEN__)
+# warning "Compiling with emscripten, without using SIMD. To use SIMD, compile with -msse2 -msimd128 -experimental-wasm-simd and -I/path/to/emsdk/upstream/lib/clang/16.0.0/include"
+# endif
+
 
 
 static inline zsv_mask_t movemask_pseudo(zsv_uc_vector v) {
 static inline zsv_mask_t movemask_pseudo(zsv_uc_vector v) {
   zsv_mask_t mask = 0, tmp = 1;
   zsv_mask_t mask = 0, tmp = 1;