Remove -t and -tt functionality from ironpython (#1131)

* Inactivate -t and -tt options, including deleting help notes

* Remove sys.flags.tabcheck

* Remove references to IndentationInconsistencySeverity

* Remove code that is no longer referenced

* Fix to check indentation every time ReadNewline is called

* initialize IndentFormat in State constructor

* Fix test failures

Co-authored-by: Stéphane Lozier <slozier@users.noreply.github.com>

Author: dc366

Src/IronPython/Compiler/Parser.cs

@@ -117,7 +117,6 @@ private static Parser CreateParserWorker(CompilerContext context, PythonOptions
             Tokenizer tokenizer = new Tokenizer(context.Errors, compilerOptions, verbatim);
 
             tokenizer.Initialize(null, reader, context.SourceUnit, SourceLocation.MinValue);
-            tokenizer.IndentationInconsistencySeverity = options.IndentationInconsistencySeverity;
 
             Parser result = new Parser(context, tokenizer, context.Errors, context.ParserSink, compilerOptions.Module);
             result._sourceReader = reader;

Src/IronPython/Compiler/Tokenizer.cs

@@ -32,7 +32,6 @@ public sealed partial class Tokenizer : TokenizerService {
         private bool _disableLineFeedLineSeparator;
         private SourceUnit _sourceUnit;
         private ErrorSink _errors;
-        private Severity _indentationInconsistencySeverity;
         private bool _endContinues;
         private List<int> _newLineLocations;
         private SourceLocation _initialLocation;
@@ -127,17 +126,6 @@ public override ErrorSink ErrorSink {
             }
         }
 
-        public Severity IndentationInconsistencySeverity {
-            get { return _indentationInconsistencySeverity; }
-            set {
-                _indentationInconsistencySeverity = value;
-
-                if (value != Severity.Ignore && _state.IndentFormat == null) {
-                    _state.IndentFormat = new StringBuilder[MaxIndent];
-                }
-            }
-        }
-
         public bool IsEndOfFile {
             get {
                 return Peek() == EOF;
@@ -1321,70 +1309,11 @@ public bool EndContinues {
             }
         }
 
-        /// <summary>
-        /// Returns whether the 
-        /// </summary>
+        // <summary>
+        // Returns whether the
+        // </summary>
         private bool ReadNewline() {
-            // Check whether we're currently scanning for inconsistent use of identation characters. If
-            // we are we'll switch to using a slower version of this method with the extra checks embedded.
-            if (IndentationInconsistencySeverity != Severity.Ignore)
-                return ReadNewlineWithChecks();
-
-            int spaces = 0;
-            while (true) {
-                int ch = NextChar();
-
-                switch (ch) {
-                    case ' ': spaces += 1; break;
-                    case '\t': spaces += 8 - (spaces % 8); break;
-                    case '\f': spaces = 0; break;
-
-                    case '#':
-                        if (_verbatim) {
-                            BufferBack();
-                            MarkTokenEnd();
-                            return true;
-                        } else {
-                            ch = ReadLine();
-                            break;
-                        }
-                    default:
-                        BufferBack();
-
-                        if (GroupingLevel > 0) {
-                            return false;
-                        }
-
-                        MarkTokenEnd();
-
-                        // if there's a blank line then we don't want to mess w/ the
-                        // indentation level - Python says that blank lines are ignored.
-                        // And if we're the last blank line in a file we don't want to
-                        // increase the new indentation level.
-                        if (ch == EOF) {
-                            if (spaces < _state.Indent[_state.IndentLevel]) {
-                                if (_sourceUnit.Kind == SourceCodeKind.InteractiveCode ||
-                                    _sourceUnit.Kind == SourceCodeKind.Statements) {
-                                    SetIndent(spaces, null);
-                                } else {
-                                    DoDedent(spaces, _state.Indent[_state.IndentLevel]);
-                                }
-                            }
-                        } else if (ch != '\n' && ch != '\r') {
-                            SetIndent(spaces, null);
-                        }
-
-                        return true;
-                }
-            }
-        }
-
-        // This is another version of ReadNewline with nearly identical semantics. The difference is
-        // that checks are made to see that indentation is used consistently. This logic is in a
-        // duplicate method to avoid inflicting the overhead of the extra logic when we're not making
-        // the checks.
-        private bool ReadNewlineWithChecks() {
-            // Keep track of the indentation format for the current line
+            // Keep track of the indentation format for the current line - may want to optimize in the future
             StringBuilder sb = new StringBuilder(80);
 
             int spaces = 0;
@@ -1394,7 +1323,7 @@ private bool ReadNewlineWithChecks() {
                 switch (ch) {
                     case ' ': spaces += 1; sb.Append(' '); break;
                     case '\t': spaces += 8 - (spaces % 8); sb.Append('\t'); break;
-                    case '\f': spaces = 0; sb.Append('\f'); break;
+                    case '\f': spaces = 0; break;
 
                     case '#':
                         if (_verbatim) {
@@ -1463,24 +1392,21 @@ private void CheckIndent(StringBuilder sb) {
                         // We've hit a difference in the way we're indenting, report it.
                         _errors.Add(_sourceUnit, Resources.InconsistentWhitespace,
                             new SourceSpan(eoln_token_end, eoln_token_end), // TODO: we can report better span - starting at the beginning of the line
-                            ErrorCodes.TabError, _indentationInconsistencySeverity
+                            ErrorCodes.TabError, Severity.Error
                         );
-
-                        // We only report problems once per module, so switch back to the fast algorithm.
-                        _indentationInconsistencySeverity = Severity.Ignore;
                     }
                 }
             }
         }
 
+
         private void SetIndent(int spaces, StringBuilder chars) {
             int current = _state.Indent[_state.IndentLevel];
             if (spaces == current) {
                 return;
             } else if (spaces > current) {
                 _state.Indent[++_state.IndentLevel] = spaces;
-                if (_state.IndentFormat != null)
-                    _state.IndentFormat[_state.IndentLevel] = chars;
+                _state.IndentFormat[_state.IndentLevel] = chars;
                 _state.PendingDedents = -1;
                 return;
             } else {
@@ -1656,8 +1582,6 @@ private struct State : IEquatable<State> {
             public int PendingDedents;
             public bool LastNewLine;        // true if the last token we emitted was a new line.
             public IncompleteString IncompleteString;
-
-            // Indentation state used only when we're reporting on inconsistent identation format.
             public StringBuilder[] IndentFormat;
 
             // grouping state
@@ -1671,15 +1595,15 @@ public State(State state) {
                 BraceLevel = state.BraceLevel;
                 PendingDedents = state.PendingDedents;
                 IndentLevel = state.IndentLevel;
-                IndentFormat = (state.IndentFormat != null) ? (StringBuilder[])state.IndentFormat.Clone() : null;
+                IndentFormat = (StringBuilder[])state.IndentFormat.Clone();
                 IncompleteString = state.IncompleteString;
             }
 
             public State(object dummy) {
                 Indent = new int[MaxIndent]; // TODO
                 LastNewLine = false;
                 BracketLevel = ParenLevel = BraceLevel = PendingDedents = IndentLevel = 0;
-                IndentFormat = null;
+                IndentFormat = new StringBuilder[MaxIndent];
                 IncompleteString = null;
             }
 

Src/IronPython/Hosting/PythonOptionsParser.cs

@@ -90,8 +90,13 @@ protected override void ParseArgument(string/*!*/ arg) {
                     LanguageSetup.Options["IgnoreEnvironment"] = ScriptingRuntimeHelpers.True;
                     break;
 
-                case "-t": LanguageSetup.Options["IndentationInconsistencySeverity"] = Severity.Warning; break;
-                case "-tt": LanguageSetup.Options["IndentationInconsistencySeverity"] = Severity.Error; break;
+                case "-t":
+                    //ignore for backwards compatibility
+                    break;
+
+                case "-tt":
+                    //ignore for backwards compatibility
+                    break;
 
                 case "-O":
                     LanguageSetup.Options["Optimize"] = ScriptingRuntimeHelpers.True;
@@ -257,8 +262,6 @@ public override void GetHelp(out string commandLine, out string[,] options, out
                 { "-S",                     "Don't imply 'import site' on initialization" },
                 { "-s",                     "Don't add user site directory to sys.path" },
                 { "-I",                     "isolate IronPython from the user's environment (implies -E and -s)" },
-                { "-t",                     "Issue warnings about inconsistent tab usage" },
-                { "-tt",                    "Issue errors for inconsistent tab usage" },
                 { "-W arg",                 "Warning control (arg is action:message:category:module:lineno) also IRONPYTHONWARNINGS=arg" },
                 { "-q",                     "don't print version and copyright messages on interactive startup" },
 

Src/IronPython/Modules/sys.cs

@@ -302,7 +302,7 @@ public static int getrecursionlimit(CodeContext/*!*/ context) {
 
         [PythonHidden, PythonType("flags"), DontMapIEnumerableToIter]
         public sealed class SysFlags : PythonTuple {
-            internal SysFlags() : base(new object[n_fields] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }) { }
+            internal SysFlags() : base(new object[n_fields] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }) { }
 
             private const int INDEX_DEBUG = 0;
             private const int INDEX_INSPECT = 1;
@@ -312,14 +312,13 @@ internal SysFlags() : base(new object[n_fields] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
             private const int INDEX_NO_USER_SITE = 5;
             private const int INDEX_NO_SITE = 6;
             private const int INDEX_IGNORE_ENVIRONMENT = 7;
-            private const int INDEX_TABCHECK = 8;
-            private const int INDEX_VERBOSE = 9;
-            private const int INDEX_UNICODE = 10;
-            private const int INDEX_BYTES_WARNING = 11;
-            private const int INDEX_QUIET = 12;
-
-            public const int n_fields = 13;
-            public const int n_sequence_fields = 13;
+            private const int INDEX_VERBOSE = 8;
+            private const int INDEX_UNICODE = 9;
+            private const int INDEX_BYTES_WARNING = 10;
+            private const int INDEX_QUIET = 11;
+
+            public const int n_fields = 12;
+            public const int n_sequence_fields = 12;
             public const int n_unnamed_fields = 0;
 
             public override string __repr__(CodeContext context) {
@@ -332,7 +331,6 @@ public override string __repr__(CodeContext context) {
                     $"{nameof(no_user_site)}={no_user_site}",
                     $"{nameof(no_site)}={no_site}",
                     $"{nameof(ignore_environment)}={ignore_environment}",
-                    $"{nameof(tabcheck)}={tabcheck}",
                     $"{nameof(verbose)}={verbose}",
                     $"{nameof(unicode)}={unicode}",
                     $"{nameof(bytes_warning)}={bytes_warning}",
@@ -383,11 +381,6 @@ public int ignore_environment {
                 internal set => _data[INDEX_IGNORE_ENVIRONMENT] = value;
             }
 
-            public int tabcheck {
-                get => (int)_data[INDEX_TABCHECK];
-                internal set => _data[INDEX_TABCHECK] = value;
-            }
-
             public int verbose {
                 get => (int)_data[INDEX_VERBOSE];
                 internal set => _data[INDEX_VERBOSE] = value;

Src/IronPython/Runtime/PythonContext.cs

@@ -719,14 +719,6 @@ private void InitializeSysFlags() {
             flags.no_user_site = PythonOptions.NoUserSite ? 1 : 0;
             flags.no_site = PythonOptions.NoSite ? 1 : 0;
             flags.ignore_environment = PythonOptions.IgnoreEnvironment ? 1 : 0;
-            switch (PythonOptions.IndentationInconsistencySeverity) {
-                case Severity.Warning:
-                    flags.tabcheck = 1;
-                    break;
-                case Severity.Error:
-                    flags.tabcheck = 2;
-                    break;
-            }
             flags.verbose = PythonOptions.Verbose ? 1 : 0;
             flags.unicode = 1;
             flags.bytes_warning = PythonOptions.BytesWarning switch {

Src/IronPython/Runtime/PythonOptions.cs

@@ -96,11 +96,6 @@ public sealed class PythonOptions : LanguageOptions {
         /// </summary>
         public bool Tracing { get; }
 
-        /// <summary> 
-        /// Severity of a warning that indentation is formatted inconsistently.
-        /// </summary>
-        public Severity IndentationInconsistencySeverity { get; }
-
         /// <summary>
         /// Forces all code to be compiled in a mode in which the code can be reliably collected by the CLR.
         /// </summary>
@@ -142,7 +137,6 @@ public PythonOptions(IDictionary<string, object> options)
             Optimize = GetOption(options, "Optimize", false);
             StripDocStrings = GetOption(options, "StripDocStrings", false);
             RecursionLimit = GetOption(options, "RecursionLimit", Int32.MaxValue);
-            IndentationInconsistencySeverity = GetOption(options, "IndentationInconsistencySeverity", Severity.Ignore);
             EnableProfiler = GetOption(options, "EnableProfiler", false);
             LightweightScopes = GetOption(options, "LightweightScopes", false);
             FullFrames = GetOption(options, "FullFrames", false);

Src/IronPythonTest/EngineTest.cs

@@ -216,7 +216,6 @@ public void ScenarioHostingHelpers() {
             options["StripDocStrings"] = true;
             options["Optimize"] = true;
             options["RecursionLimit"] = 42;
-            options["IndentationInconsistencySeverity"] = Severity.Warning;
             options["WarningFilters"] = new string[] { "warnonme" };
 
             ScriptEngine engine1 = Python.CreateEngine();
@@ -276,7 +275,6 @@ private void TestEngine(ScriptEngine scriptEngine, Dictionary<string, object> op
                 Assert.AreEqual(po.StripDocStrings, true);
                 Assert.AreEqual(po.Optimize, true);
                 Assert.AreEqual(po.RecursionLimit, 42);
-                Assert.AreEqual(po.IndentationInconsistencySeverity, Severity.Warning);
                 Assert.AreEqual(po.WarningFilters[0], "warnonme");
             }
 

Tests/test_regressions.py

@@ -589,13 +589,13 @@ def cp22692_helper(self, source, flags):
 
     def test_cp22692(self):
         self.assertEqual(self.cp22692_helper("if 1:", 0x200),
-                [SyntaxError, IndentationError if is_cli else SyntaxError])
+                [IndentationError if sys.version_info >= (3,9) else SyntaxError, IndentationError if sys.version_info >= (3,9) else SyntaxError])
         self.assertEqual(self.cp22692_helper("if 1:", 0),
-                [SyntaxError, IndentationError if is_cli else SyntaxError])
+                [IndentationError if sys.version_info >= (3,9) else SyntaxError, IndentationError if sys.version_info >= (3,9) else SyntaxError])
         self.assertEqual(self.cp22692_helper("if 1:\n  if 1:", 0x200),
-                [IndentationError if is_cli else SyntaxError, IndentationError if is_cli else SyntaxError])
+                [IndentationError if is_cli or sys.version_info >= (3,9) else SyntaxError, IndentationError if is_cli or sys.version_info >= (3,9) else SyntaxError])
         self.assertEqual(self.cp22692_helper("if 1:\n  if 1:", 0),
-                [IndentationError if is_cli else SyntaxError, IndentationError if is_cli else SyntaxError])
+                [IndentationError if is_cli or sys.version_info >= (3,9) else SyntaxError, IndentationError if is_cli or sys.version_info >= (3,9) else SyntaxError])
 
     @skipUnlessIronPython()
     def test_cp23545(self):

Tests/test_stdconsole.py

@@ -229,12 +229,8 @@ def test_t(self):
             f.write("if (1):\n\tpass\n        pass\nprint('OK')\n")
 
         msg = "inconsistent use of tabs and spaces in indentation"
-        if is_cli: # https://github.com/IronLanguages/ironpython3/issues/982
-            self.TestCommandLine((tmpscript, ), "OK\n")
-            self.TestCommandLine(("-t", tmpscript), ("firstline", "%s:3: SyntaxWarning: %s\n"  % (tmpscript, msg, )), 0)
-        else:
-            self.TestCommandLine((tmpscript, ), ("lastline", "TabError: " + msg + "\n"), 1)
-            self.TestCommandLine(("-t", tmpscript), ("lastline", "TabError: " + msg + "\n"), 1)
+        self.TestCommandLine((tmpscript, ), ("lastline", "TabError: " + msg + "\n"), 1)
+        self.TestCommandLine(("-t", tmpscript), ("lastline", "TabError: " + msg + "\n"), 1)
         self.TestCommandLine(("-tt", tmpscript), ("lastline", "TabError: " + msg + "\n"), 1)
 
         tmpscript = os.path.join(self.tmpdir, "funcdef.py")