[pull] master from ruby:master

prism/templates/template.rb

@@ -690,9 +690,9 @@ def locals
       "javascript/src/deserialize.js",
       "javascript/src/nodes.js",
       "javascript/src/visitor.js",
-      "java/api/target/generated-sources/java/org/ruby_lang/prism/Loader.java",
-      "java/api/target/generated-sources/java/org/ruby_lang/prism/Nodes.java",
-      "java/api/target/generated-sources/java/org/ruby_lang/prism/AbstractNodeVisitor.java",
+      "java/api/src/main/java-templates/org/ruby_lang/prism/Loader.java",
+      "java/api/src/main/java-templates/org/ruby_lang/prism/Nodes.java",
+      "java/api/src/main/java-templates/org/ruby_lang/prism/AbstractNodeVisitor.java",
       "lib/prism/compiler.rb",
       "lib/prism/dispatcher.rb",
       "lib/prism/dot_visitor.rb",

test/prism/newline_test.rb

@@ -20,6 +20,7 @@ class NewlineTest < TestCase
       ruby/find_fixtures.rb
       ruby/find_test.rb
       ruby/parser_test.rb
+      ruby/ripper_test.rb
       ruby/ruby_parser_test.rb
     ]
 

test/prism/ruby/ripper_test.rb

@@ -136,8 +136,93 @@ def test_lex_ignored_missing_heredoc_end
       end
     end
 
-    UNSUPPORTED_EVENTS = %i[comma ignored_nl kw label_end lbrace lbracket lparen nl op rbrace rbracket rparen semicolon sp words_sep ignored_sp]
+    # Events that are currently not emitted
+    UNSUPPORTED_EVENTS = %i[comma ignored_nl label_end lbrace lbracket lparen nl op rbrace rbracket rparen semicolon sp words_sep ignored_sp]
     SUPPORTED_EVENTS = Translation::Ripper::EVENTS - UNSUPPORTED_EVENTS
+    # Events that assert against their line/column
+    CHECK_LOCATION_EVENTS = %i[kw]
+    IGNORE_FOR_SORT_EVENTS = %i[
+      stmts_new stmts_add bodystmt void_stmt
+      args_new args_add args_add_star args_add_block arg_paren method_add_arg
+      mlhs_new mlhs_add_star
+      word_new words_new symbols_new qwords_new qsymbols_new xstring_new regexp_new
+      words_add symbols_add qwords_add qsymbols_add
+      regexp_end tstring_end heredoc_end
+      call command fcall vcall
+      field aref_field var_field var_ref block_var ident params
+      string_content heredoc_dedent unary binary dyna_symbol
+      comment magic_comment embdoc embdoc_beg embdoc_end arg_ambiguous
+    ]
+    SORT_IGNORE = {
+      aref: [
+        "blocks.txt",
+        "command_method_call.txt",
+        "whitequark/ruby_bug_13547.txt",
+      ],
+      assoc_new: [
+        "case_in_hash_key.txt",
+        "whitequark/parser_bug_525.txt",
+        "whitequark/ruby_bug_11380.txt",
+      ],
+      bare_assoc_hash: [
+        "case_in_hash_key.txt",
+        "method_calls.txt",
+        "whitequark/parser_bug_525.txt",
+        "whitequark/ruby_bug_11380.txt",
+      ],
+      brace_block: [
+        "super.txt",
+        "unparser/corpus/literal/super.txt"
+      ],
+      command_call: [
+        "blocks.txt",
+        "case_in_hash_key.txt",
+        "seattlerb/block_call_dot_op2_cmd_args_do_block.txt",
+        "seattlerb/block_call_operation_colon.txt",
+        "seattlerb/block_call_operation_dot.txt",
+      ],
+      const_path_field: [
+        "seattlerb/const_2_op_asgn_or2.txt",
+        "seattlerb/const_op_asgn_or.txt",
+        "whitequark/const_op_asgn.txt",
+      ],
+      const_path_ref: ["unparser/corpus/literal/defs.txt"],
+      do_block: ["whitequark/super_block.txt"],
+      embexpr_end: ["seattlerb/str_interp_ternary_or_label.txt"],
+      rest_param: ["whitequark/send_lambda.txt"],
+      top_const_field: [
+        "seattlerb/const_3_op_asgn_or.txt",
+        "seattlerb/const_op_asgn_and1.txt",
+        "seattlerb/const_op_asgn_and2.txt",
+        "whitequark/const_op_asgn.txt",
+      ],
+      mlhs_paren: ["unparser/corpus/literal/for.txt"],
+      mlhs_add: [
+        "whitequark/for_mlhs.txt",
+      ],
+      kw: [
+        "defined.txt",
+        "for.txt",
+        "seattlerb/block_kw__required.txt",
+        "seattlerb/case_in_42.txt",
+        "seattlerb/case_in_67.txt",
+        "seattlerb/case_in_86_2.txt",
+        "seattlerb/case_in_86.txt",
+        "seattlerb/case_in_hash_pat_paren_true.txt",
+        "seattlerb/flip2_env_lvar.txt",
+        "unless.txt",
+        "unparser/corpus/semantic/and.txt",
+        "whitequark/class.txt",
+        "whitequark/find_pattern.txt",
+        "whitequark/pattern_matching_hash.txt",
+        "whitequark/pattern_matching_implicit_array_match.txt",
+        "whitequark/pattern_matching_ranges.txt",
+        "whitequark/super_block.txt",
+        "write_command_operator.txt",
+      ],
+    }
+    SORT_IGNORE.default = []
+    SORT_EVENTS = SUPPORTED_EVENTS - IGNORE_FOR_SORT_EVENTS
 
     module Events
       attr_reader :events
@@ -147,9 +232,20 @@ def initialize(...)
         @events = []
       end
 
+      def sorted_events
+        @events.select do |e,|
+          next false if e == :kw && @events.any? { |e,| e == :if_mod || e == :while_mod || e == :until_mod || e == :rescue || e == :rescue_mod || e == :while || e == :ensure }
+          SORT_EVENTS.include?(e) && !SORT_IGNORE[e].include?(filename)
+        end
+      end
+
       SUPPORTED_EVENTS.each do |event|
         define_method(:"on_#{event}") do |*args|
-          @events << [event, *args.map(&:to_s)]
+          if CHECK_LOCATION_EVENTS.include?(event)
+            @events << [event, lineno, column, *args.map(&:to_s)]
+          else
+            @events << [event, *args.map(&:to_s)]
+          end
           super(*args)
         end
       end
@@ -177,12 +273,14 @@ class ObjectEvents < Translation::Ripper
         object_events = ObjectEvents.new(source)
         assert_nothing_raised { object_events.parse }
 
-        ripper = RipperEvents.new(source)
-        prism = PrismEvents.new(source)
+        ripper = RipperEvents.new(source, fixture.path)
+        prism = PrismEvents.new(source, fixture.path)
         ripper.parse
         prism.parse
-        # This makes sure that the content is the same. Ordering is not correct for now.
+        # Check that the same events are emitted, regardless of order
         assert_equal(ripper.events.sort, prism.events.sort)
+        # Check a subset of events against the correct order
+        assert_equal(ripper.sorted_events, prism.sorted_events)
       end
     end
 

zjit/src/codegen.rs

@@ -2549,7 +2549,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard
 
         asm.cmp(klass, Opnd::Value(expected_class));
         asm.jne(jit, side_exit);
-    } else if guard_type.is_subtype(types::String) {
+    } else if guard_type.is_subtype(types::TypedTData) {
         let side = side_exit(jit, state, GuardType(guard_type));
 
         // Check special constant
@@ -2560,13 +2560,15 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard
         asm.cmp(val, Qfalse.into());
         asm.je(jit, side.clone());
 
+        // Check the builtin type and RUBY_TYPED_FL_IS_TYPED_DATA with mask and compare
         let val = asm.load_mem(val);
-
         let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS));
-        let tag   = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64));
-        asm.cmp(tag, Opnd::UImm(RUBY_T_STRING as u64));
+        let mask     = RUBY_T_MASK.to_usize() | RUBY_TYPED_FL_IS_TYPED_DATA.to_usize();
+        let expected = RUBY_T_DATA.to_usize() | RUBY_TYPED_FL_IS_TYPED_DATA.to_usize();
+        let masked = asm.and(flags, mask.into());
+        asm.cmp(masked, expected.into());
         asm.jne(jit, side);
-    } else if guard_type.is_subtype(types::Array) {
+    } else if let Some(builtin_type) = guard_type.builtin_type_equivalent() {
         let side = side_exit(jit, state, GuardType(guard_type));
 
         // Check special constant
@@ -2577,11 +2579,11 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard
         asm.cmp(val, Qfalse.into());
         asm.je(jit, side.clone());
 
+        // Mask and check the builtin type
         let val = asm.load_mem(val);
-
         let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS));
         let tag   = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64));
-        asm.cmp(tag, Opnd::UImm(RUBY_T_ARRAY as u64));
+        asm.cmp(tag, Opnd::UImm(builtin_type as u64));
         asm.jne(jit, side);
     } else if guard_type.bit_equal(types::HeapBasicObject) {
         let side_exit = side_exit(jit, state, GuardType(guard_type));

zjit/src/codegen_tests.rs

@@ -3623,6 +3623,97 @@ fn test_attr_accessor_getivar() {
     assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[4, 4]");
 }
 
+#[test]
+fn test_getivar_t_data_then_string() {
+    // This is a regression test for a type confusion miscomp where
+    // we end up reading the fields object using an offset off of a
+    // string, assuming that it has a the same layout as a T_DATA object.
+    // At the time of writing the fields object of strings are stored
+    // in a global table, out-of-line of each string.
+    // The string and the thread end up sharing one shape ID.
+    set_call_threshold(2);
+    eval(r#"
+      module GetThousand
+        def test = @var1000
+      end
+      class Thread
+        include GetThousand
+      end
+      class String
+        include GetThousand
+      end
+      OBJ = Thread.new { }
+      OBJ.join
+      STR = +''
+      (0..1000).each do |i|
+        ivar_name = :"@var#{i}"
+        OBJ.instance_variable_set(ivar_name, i)
+        STR.instance_variable_set(ivar_name, i)
+      end
+      OBJ.test; OBJ.test # profile and compile for Thread (T_DATA)
+    "#);
+    assert_snapshot!(assert_compiles("[STR.test, STR.test]"), @"[1000, 1000]");
+}
+
+#[test]
+fn test_getivar_t_object_then_string() {
+    // This test construct an object and a string that have the same set of ivars.
+    // They wouldn't share the same shape ID, though, and we rely on this fact in
+    // our guards.
+    set_call_threshold(2);
+    eval(r#"
+      module GetThousand
+        def test = @var1000
+      end
+      class MyObject
+        include GetThousand
+      end
+      class String
+        include GetThousand
+      end
+      OBJ = MyObject.new
+      STR = +''
+      (0..1000).each do |i|
+        ivar_name = :"@var#{i}"
+        OBJ.instance_variable_set(ivar_name, i)
+        STR.instance_variable_set(ivar_name, i)
+      end
+      OBJ.test; OBJ.test # profile and compile for MyObject
+    "#);
+    assert_snapshot!(assert_compiles("[STR.test, STR.test]"), @"[1000, 1000]");
+}
+
+#[test]
+fn test_getivar_t_class_then_string() {
+    // This is a regression test for a type confusion miscomp where
+    // we end up reading the fields object using an offset off of a
+    // string, assuming that it has a the same layout as a T_CLASS object.
+    // At the time of writing the fields object of strings are stored
+    // in a global table, out-of-line of each string.
+    // The string and the class end up sharing one shape ID.
+    set_call_threshold(2);
+    eval(r#"
+      module GetThousand
+        def test = @var1000
+      end
+      class MyClass
+        extend GetThousand
+      end
+      class String
+        include GetThousand
+      end
+      STR = +''
+      (0..1000).each do |i|
+        ivar_name = :"@var#{i}"
+        MyClass.instance_variable_set(ivar_name, i)
+        STR.instance_variable_set(ivar_name, i)
+      end
+      p MyClass.test; p MyClass.test # profile and compile for MyClass
+      p STR.test
+    "#);
+    assert_snapshot!(assert_compiles("[STR.test, STR.test]"), @"[1000, 1000]");
+}
+
 #[test]
 fn test_attr_accessor_setivar() {
     eval("

zjit/src/cruby.rs

@@ -604,6 +604,7 @@ impl VALUE {
         unsafe { rb_jit_class_fields_embedded_p(self) }
     }
 
+    /// Typed `T_DATA` made from `TypedData_Make_Struct()` (e.g. Thread, ARGF)
     pub fn typed_data_p(self) -> bool {
         !self.special_const_p() &&
             self.builtin_type() == RUBY_T_DATA &&