infinite loop when adjusting vertical split size with mouse with smoothscroll + number + showbreak + scrolloff

[tomtomjhj]

Steps to reproduce

1. Run vim as follows, in 80 cells wide terminal.

vim --clean -S <(tee << 'EOF'
    set mouse=a smoothscroll number showbreak=> scrolloff=2
    vnew
    call setline(1, 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.')
    call setline(2, 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.')
    call setline(3, 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.')
    exe "normal! \<C-e>"
EOF
)

2. click and drag the vertical window separator all the way to the left. Then vim falls into an infinite loop

Expected behaviour

no infinite loop

Version of Vim

9.1.0407

Environment

ubuntu 22.04

Logs and stack traces

0x000055b869de8940 in cursor_correct_sms () at move.c:1642
1642        while (so_cols > size && so_cols - width2 >= width1)
(gdb) bt
#0  0x000055b869de8940 in cursor_correct_sms () at move.c:1642
#1  0x000055b869deb7a4 in cursor_correct_sms () at move.c:2755
#2  scroll_cursor_bot (min_scroll=1, set_topbot=set_topbot@entry=0) at move.c:2787
#3  0x000055b869debebd in update_topline () at move.c:495
#4  0x000055b869dec19a in curs_columns (may_scroll=may_scroll@entry=1) at move.c:1149
#5  0x000055b869f33506 in win_new_width (wp=0x55b86a4070a0, width=<optimized out>) at window.c:7245
#6  0x000055b869f3356c in frame_new_width (topfrp=topfrp@entry=0x55b86a409940, width=5, leftfirst=leftfirst@entry=0, wfw=wfw@entry=0) at window.c:4049
#7  0x000055b869f367b3 in win_drag_vsep_line (dragwin=<optimized out>, offset=1, offset@entry=-1) at window.c:6922
#8  0x000055b869de5e4a in jump_to_mouse (flags=7, inclusive=<optimized out>, which_button=which_button@entry=0) at mouse.c:1917
#9  0x000055b869de68bb in do_mouse (oap=0x7ffecb80d930, c=<optimized out>, dir=-1, count=1, fixindent=0) at mouse.c:708
#10 0x000055b869df6768 in normal_cmd (oap=oap@entry=0x7ffecb80d930, toplevel=toplevel@entry=1) at normal.c:949
#11 0x000055b869f8a39a in main_loop (cmdwin=cmdwin@entry=0, noexmode=noexmode@entry=0) at main.c:1562
#12 0x000055b869f8b643 in vim_main2 () at main.c:895
#13 0x00007fdb4a684d90 in __libc_start_call_main (main=main@entry=0x55b869cd8ef0 <main>, argc=argc@entry=4, argv=argv@entry=0x7ffecb80dbb8) at ../sysdeps/nptl/libc_start_call_main.h:58
#14 0x00007fdb4a684e40 in __libc_start_main_impl
     (main=0x55b869cd8ef0 <main>, argc=4, argv=0x7ffecb80dbb8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffecb80dba8) at ../csu/libc-start.c:392
#15 0x000055b869cdad85 in _start ()

[chrisbra]

Hm,
I see two problems:

1. width1 is zero in cursor_correct_sms() and more importantly,
2. size which get's assigned from win_linetabsize() overflows and gets negative. Unfortunately, win_linetabsize() is a very central function, so changing it may affect many call sites (any may cause other issues which I do not foresee now) :/

The following patch fixes it for me (and it doesn't seem to cause any regressions), but I am not hundert percent sure.

diff --git a/src/charset.c b/src/charset.c
index 470698f0e..e47b679f5 100644
--- a/src/charset.c
+++ b/src/charset.c
@@ -770,6 +770,9 @@ linetabsize_col(int startcol, char_u *s)
     while (*cts.cts_ptr != NUL)
        cts.cts_vcol += lbr_chartabsize_adv(&cts);
     clear_chartabsize_arg(&cts);
+    // overflow, e.g. because of a small window when the start of the line is no longer visible
+    if (cts.cts_vcol < 0)
+       return MAXCOL;
     return (int)cts.cts_vcol;
 }

@@ -784,6 +787,9 @@ win_linetabsize(win_T *wp, linenr_T lnum, char_u *line, colnr_T len)
     init_chartabsize_arg(&cts, wp, lnum, 0, line, line);
     win_linetabsize_cts(&cts, len);
     clear_chartabsize_arg(&cts);
+    // overflow, e.g. because of a small window when the start of the line is no longer visible
+    if (cts.cts_vcol < 0)
+       return MAXCOL;
     return (int)cts.cts_vcol;
 }

diff --git a/src/move.c b/src/move.c
index 3e589caae..fe820ef07 100644
--- a/src/move.c
+++ b/src/move.c
@@ -1630,6 +1630,7 @@ static void cursor_correct_sms(void)
     int            width2 = width1 + curwin_col_off2();
     int            so_cols = so == 0 ? 0 : width1 + (so - 1) * width2;
     int            space_cols = (curwin->w_height - 1) * width2;
+    int            overlap, top, bot;
     int            size = so == 0 ? 0 : win_linetabsize(curwin, curwin->w_topline,
                                    ml_get(curwin->w_topline), (colnr_T)MAXCOL);

@@ -1639,16 +1640,16 @@ static void cursor_correct_sms(void)
        so_cols = space_cols / 2;  // Not enough room: put cursor in the middle.

     // Not enough screen lines in topline: ignore 'scrolloff'.
-    while (so_cols > size && so_cols - width2 >= width1)
+    while (so_cols > size && so_cols - width2 >= width1 && width1 > 0 && size > 0)
        so_cols -= width2;
     if (so_cols >= width1 && so_cols > size)
        so_cols -= width1;

     // If there is no marker or we have non-zero scrolloff, just ignore it.
-    int overlap = (curwin->w_skipcol == 0 || so_cols != 0) ? 0
+    overlap = (curwin->w_skipcol == 0 || so_cols != 0) ? 0
                                            : sms_marker_overlap(curwin, -1);
-    int top = curwin->w_skipcol + overlap + so_cols;
-    int bot = curwin->w_skipcol + width1 + (curwin->w_height - 1) * width2
+    top = curwin->w_skipcol + overlap + so_cols;
+    bot = curwin->w_skipcol + width1 + (curwin->w_height - 1) * width2
                                                                    - so_cols;
     validate_virtcol();
     colnr_T col = curwin->w_virtcol;

@luukvbaal what do you think?

[luukvbaal]

Yeah avoiding to return overflown value seems desirable. I also don't foresee it causing any issues when returning MAXCOL. I don't think the size > 0 check still necessary in that case, and the width1 > 0 check probably makes more sense at the if statement below(and others that increment with width1).