Dumb terminal when piping input to console app disallows ANSI formatt…

…ing, fixes #299
jline · Nov 7, 2020 · 404565b · 404565b
1 parent fe1f271
commit 404565b
Show file tree

Hide file tree

Showing 17 changed files with 105 additions and 20 deletions.
diff --git a/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/JansiNativePty.java b/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/JansiNativePty.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2017, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -172,4 +172,8 @@ protected static FileDescriptor newDescriptor(int fd) {
         }
     }
 
+    public static boolean isConsoleOutput() {
+        return CLibrary.isatty(1) == 1;
+    }
+
 }
diff --git a/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/JansiSupportImpl.java b/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/JansiSupportImpl.java
@@ -136,4 +136,8 @@ public boolean isWindowsConsole() {
         return JansiWinSysTerminal.isWindowsConsole();
     }
 
+    @Override
+    public boolean isConsoleOutput(boolean pty) {
+        return pty ? JansiNativePty.isConsoleOutput() : JansiWinSysTerminal.isConsoleOutput();
+    }
 }
diff --git a/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/win/JansiWinSysTerminal.java b/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/win/JansiWinSysTerminal.java
@@ -38,6 +38,8 @@
 import static org.fusesource.jansi.internal.Kernel32.readConsoleInputHelper;
 
 public class JansiWinSysTerminal extends AbstractWindowsTerminal {
+    private static final long consoleOut = GetStdHandle(STD_OUTPUT_HANDLE);
+    private static final long consoleIn = GetStdHandle(STD_INPUT_HANDLE);
 
     public static JansiWinSysTerminal createTerminal(String name, String type, boolean ansiPassThrough, Charset encoding, int codepage, boolean nativeSignals, SignalHandler signalHandler, boolean paused) throws IOException {
         Writer writer;
@@ -47,11 +49,8 @@ public static JansiWinSysTerminal createTerminal(String name, String type, boole
             }
             writer = new JansiWinConsoleWriter();
         } else {
-            long consoleOut = GetStdHandle(STD_OUTPUT_HANDLE);
-            long consoleIn = GetStdHandle(STD_INPUT_HANDLE);
             int[] mode = new int[1];
-            if (Kernel32.GetConsoleMode(consoleOut, mode) == 0
-                  || Kernel32.GetConsoleMode(consoleIn, mode) == 0) {
+            if (Kernel32.GetConsoleMode(consoleOut, mode) == 0 ) {
                 throw new IOException("Failed to get console mode: " + getLastErrorMessage());
             }
             if (Kernel32.SetConsoleMode(consoleOut, mode[0] | AbstractWindowsTerminal.ENABLE_VIRTUAL_TERMINAL_PROCESSING) != 0) {
@@ -70,6 +69,9 @@ public static JansiWinSysTerminal createTerminal(String name, String type, boole
                 }
                 writer = new WindowsAnsiWriter(new BufferedWriter(new JansiWinConsoleWriter()));
             }
+            if (Kernel32.GetConsoleMode(consoleIn, mode) == 0) {
+                throw new IOException("Failed to get console mode: " + getLastErrorMessage());
+            }
         }
         JansiWinSysTerminal terminal = new JansiWinSysTerminal(writer, name, type, encoding, codepage, nativeSignals, signalHandler);
         // Start input pump thread
@@ -80,12 +82,15 @@ public static JansiWinSysTerminal createTerminal(String name, String type, boole
     }
 
     public static boolean isWindowsConsole() {
-        long consoleOut = GetStdHandle(STD_OUTPUT_HANDLE);
-        long consoleIn = GetStdHandle(STD_INPUT_HANDLE);
         int[] mode = new int[1];
         return Kernel32.GetConsoleMode(consoleOut, mode) != 0 && Kernel32.GetConsoleMode(consoleIn, mode) != 0;
     }
 
+    public static boolean isConsoleOutput() {
+        int[] mode = new int[1];
+        return Kernel32.GetConsoleMode(consoleOut, mode) != 0;
+    }
+
     JansiWinSysTerminal(Writer writer, String name, String type, Charset encoding, int codepage, boolean nativeSignals, SignalHandler signalHandler) throws IOException {
         super(writer, name, type, encoding, codepage, nativeSignals, signalHandler);
     }

diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/JnaNativePty.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/JnaNativePty.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -147,4 +147,18 @@ public String toString() {
         return "JnaNativePty[" + getName() + "]";
     }
 
+    public static boolean isConsoleOutput() {
+        if (Platform.isMac()) {
+            return OsXNativePty.isatty(1) == 1;
+        } else if (Platform.isLinux()) {
+            return LinuxNativePty.isatty(1) == 1;
+        } else if (Platform.isSolaris()) {
+            return SolarisNativePty.isatty(1) == 1;
+        } else if (Platform.isFreeBSD()) {
+            return FreeBsdNativePty.isatty(1) == 1;
+        } else {
+            return false;
+        }
+    }
+
 }
diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/JnaSupportImpl.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/JnaSupportImpl.java
@@ -44,4 +44,9 @@ public boolean isWindowsConsole() {
         return JnaWinSysTerminal.isWindowsConsole();
     }
 
+    @Override
+    public boolean isConsoleOutput(boolean pty) {
+        return pty ? JnaNativePty.isConsoleOutput() : JnaWinSysTerminal.isConsoleOutput();
+    }
+
 }
diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/freebsd/CLibrary.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/freebsd/CLibrary.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -31,6 +31,8 @@ public interface CLibrary extends com.sun.jna.Library {
 
     void ioctl(int fd, long cmd, winsize data) throws LastErrorException;
 
+    int isatty(int fd);
+
     void ttyname_r(int fd, byte[] buf, int len) throws LastErrorException;
 
     class winsize extends Structure {

diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/freebsd/FreeBsdNativePty.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/freebsd/FreeBsdNativePty.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -96,4 +96,8 @@ public void setSize(Size size) throws IOException {
         C_LIBRARY.ioctl(getSlave(), TIOCSWINSZ, sz);
     }
 
+    public static int isatty(int fd) {
+        return C_LIBRARY.isatty(fd);
+    }
+
 }
diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/linux/CLibrary.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/linux/CLibrary.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -31,6 +31,8 @@ public interface CLibrary extends com.sun.jna.Library {
 
     void ioctl(int fd, int cmd, winsize data) throws LastErrorException;
 
+    int isatty(int fd);
+
     void ttyname_r(int fd, byte[] buf, int len) throws LastErrorException;
 
     class winsize extends Structure {

diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/linux/LinuxNativePty.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/linux/LinuxNativePty.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -102,4 +102,7 @@ public void setSize(Size size) throws IOException {
         C_LIBRARY.ioctl(getSlave(), TIOCSWINSZ, sz);
     }
 
+    public static int isatty(int fd) {
+        return C_LIBRARY.isatty(fd);
+    }
 }
diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/osx/CLibrary.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/osx/CLibrary.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -32,6 +32,8 @@ public interface CLibrary extends com.sun.jna.Library {
 
     void ioctl(int fd, NativeLong cmd, winsize data) throws LastErrorException;
 
+    int isatty(int fd);
+
     void ttyname_r(int fd, byte[] buf, int len) throws LastErrorException;
 
     void openpty(int[] master, int[] slave, byte[] name, termios t, winsize s) throws LastErrorException;

diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/osx/OsXNativePty.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/osx/OsXNativePty.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -89,4 +89,8 @@ public void setSize(Size size) throws IOException {
         C_LIBRARY.ioctl(getSlave(), new NativeLong(TIOCSWINSZ), sz);
     }
 
+    public static int isatty(int fd) {
+        return C_LIBRARY.isatty(fd);
+    }
+
 }
diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/solaris/CLibrary.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/solaris/CLibrary.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -31,7 +31,7 @@ public interface CLibrary extends com.sun.jna.Library {
 
     void ioctl(int fd, long cmd, winsize data) throws LastErrorException;
 
-//    int isatty(int fd);
+    int isatty(int fd);
 
     void ttyname_r(int fd, byte[] buf, int len) throws LastErrorException;
 

diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/solaris/SolarisNativePty.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/solaris/SolarisNativePty.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2016, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -88,4 +88,8 @@ public void setSize(Size size) throws IOException {
         C_LIBRARY.ioctl(getSlave(), TIOCSWINSZ, sz);
     }
 
+    public static int isatty(int fd) {
+        return C_LIBRARY.isatty(fd);
+    }
+
 }
diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/JnaWinSysTerminal.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/JnaWinSysTerminal.java
@@ -77,6 +77,16 @@ public static boolean isWindowsConsole() {
         }
     }
 
+    public static boolean isConsoleOutput() {
+        try {
+            IntByReference mode = new IntByReference();
+            Kernel32.INSTANCE.GetConsoleMode(consoleOut, mode);
+            return true;
+        } catch (LastErrorException e) {
+            return false;
+        }
+    }
+
     JnaWinSysTerminal(Writer writer, String name, String type, Charset encoding, int codepage, boolean nativeSignals, SignalHandler signalHandler) throws IOException {
         super(writer, name, type, encoding, codepage, nativeSignals, signalHandler);
         strings.put(InfoCmp.Capability.key_mouse, "\\E[M");

diff --git a/terminal/src/main/java/org/jline/terminal/TerminalBuilder.java b/terminal/src/main/java/org/jline/terminal/TerminalBuilder.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2019, the original author or authors.
+ * Copyright (c) 2002-2020, the original author or authors.
  *
  * This software is distributable under the BSD license. See the terms of the
  * BSD license in the documentation provided with this software.
@@ -323,12 +323,14 @@ private Terminal doBuild() throws IOException {
                 Log.warn("Attributes and size fields are ignored when creating a system terminal");
             }
             IllegalStateException exception = new IllegalStateException("Unable to create a system terminal");
+            boolean consoleOutput = false;
             Terminal terminal = null;
             if (OSUtils.IS_WINDOWS) {
                 boolean ansiPassThrough = OSUtils.IS_CONEMU;
                 if (terminal == null && jna) {
                     try {
                         JnaSupport support = load(JnaSupport.class);
+                        consoleOutput = support.isConsoleOutput();
                         terminal = support.winSysTerminal(name, type, ansiPassThrough, encoding, codepage, nativeSignals, signalHandler, paused);
                     } catch (Throwable t) {
                         Log.debug("Error creating JNA based terminal: ", t.getMessage(), t);
@@ -338,6 +340,7 @@ private Terminal doBuild() throws IOException {
                 if (terminal == null && jansi) {
                     try {
                         JansiSupport support = load(JansiSupport.class);
+                        consoleOutput = support.isConsoleOutput();
                         terminal = support.winSysTerminal(name, type, ansiPassThrough, encoding, codepage, nativeSignals, signalHandler, paused);
                     } catch (Throwable t) {
                         Log.debug("Error creating JANSI based terminal: ", t.getMessage(), t);
@@ -369,7 +372,9 @@ private Terminal doBuild() throws IOException {
             } else {
                 if (terminal == null && jna) {
                     try {
-                        Pty pty = load(JnaSupport.class).current();
+                        JnaSupport support = load(JnaSupport.class);
+                        consoleOutput = support.isConsoleOutput(true);
+                        Pty pty = support.current();
                         terminal = new PosixSysTerminal(name, type, pty, encoding, nativeSignals, signalHandler);
                     } catch (Throwable t) {
                         // ignore
@@ -379,7 +384,9 @@ private Terminal doBuild() throws IOException {
                 }
                 if (terminal == null && jansi) {
                     try {
-                        Pty pty = load(JansiSupport.class).current();
+                        JansiSupport support = load(JansiSupport.class);
+                        consoleOutput = support.isConsoleOutput(true);
+                        Pty pty = support.current();
                         terminal = new PosixSysTerminal(name, type, pty, encoding, nativeSignals, signalHandler);
                     } catch (Throwable t) {
                         Log.debug("Error creating JANSI based terminal: ", t.getMessage(), t);
@@ -426,6 +433,9 @@ public void run() {
                     String command = getParentProcessCommand();
                     color = command != null && command.contains("idea");
                 }
+                if (!color) {
+                    color = consoleOutput && System.getenv("TERM") != null;
+                }
                 if (!color && dumb == null) {
                     if (Log.isDebugEnabled()) {
                         Log.warn("Creating a dumb terminal", exception);

diff --git a/terminal/src/main/java/org/jline/terminal/spi/JansiSupport.java b/terminal/src/main/java/org/jline/terminal/spi/JansiSupport.java
@@ -27,4 +27,10 @@ public interface JansiSupport {
 
     boolean isWindowsConsole();
 
+    default boolean isConsoleOutput() {
+        return isConsoleOutput(false);
+    }
+
+    boolean isConsoleOutput(boolean pty);
+
 }
diff --git a/terminal/src/main/java/org/jline/terminal/spi/JnaSupport.java b/terminal/src/main/java/org/jline/terminal/spi/JnaSupport.java
@@ -27,4 +27,10 @@ public interface JnaSupport {
 
     boolean isWindowsConsole();
 
+    default boolean isConsoleOutput() {
+        return isConsoleOutput(false);
+    }
+
+    boolean isConsoleOutput(boolean pty);
+
 }