[pull] master from ruby:master

.gdbinit

@@ -983,49 +983,35 @@ define print_lineno
   set $index = 0
   set $size = $iseq->body->insns_info.size
   set $table = $iseq->body->insns_info.body
-  set $positions = $iseq->body->insns_info.positions
   #printf "size: %d\n", $size
   if $size == 0
   else
   if $size == 1
     printf "%d", $table[0].line_no
   else
-    if $positions
-      # get_insn_info_linear_search
-      set $index = 1
-      while $index < $size
-        #printf "table[%d]: position: %d, line: %d, pos: %d\n", $i, $positions[$i], $table[$i].line_no, $pos
-        if $positions[$index] > $pos
-          loop_break
-        end
-        set $index = $index + 1
-        if $positions[$index] == $pos
-          loop_break
-        end
-      end
+    # get_insn_info_succinct_bitvector (VM_INSN_INFO_TABLE_IMPL == 2).
+    # insns_info.positions and insns_info.succ_index_table share a union, and
+    # an iseq running on the stack always has succ_index_table.
+    set $sd = $iseq->body->insns_info.positions_or_succ_index_table.succ_index_table
+    set $immediate_table_size = sizeof($sd->imm_part) / sizeof(uint64_t) * 9
+    if $pos < $immediate_table_size
+      set $i = $pos / 9
+      set $j = $pos % 9
+      set $index = ((int)($sd->imm_part[$i] >> ($j * 7))) & 0x7f
     else
-      # get_insn_info_succinct_bitvector
-      set $sd = $iseq->body->insns_info.succ_index_table
-      set $immediate_table_size = sizeof($sd->imm_part) / sizeof(uint64_t) * 9
-      if $pos < $immediate_table_size
-        set $i = $pos / 9
-        set $j = $pos % 9
-        set $index = ((int)($sd->imm_part[$i] >> ($j * 7))) & 0x7f
-      else
-        set $block_index = ($pos - $immediate_table_size) / 512
-        set $block = &$sd->succ_part[$block_index]
-        set $block_bit_index = ($pos - $immediate_table_size) % 512
-        set $small_block_index = $block_bit_index / 64
-        set $small_block_popcount = $small_block_index == 0 ? 0 : (((int)($block->small_block_ranks >> (($small_block_index - 1) * 9))) & 0x1ff)
-        set $x = $block->bits[$small_block_index] << (63 - $block_bit_index % 64)
-        set $x = ($x & 0x5555555555555555) + ($x >> 1 & 0x5555555555555555)
-        set $x = ($x & 0x3333333333333333) + ($x >> 2 & 0x3333333333333333)
-        set $x = ($x & 0x0707070707070707) + ($x >> 4 & 0x0707070707070707)
-        set $x = ($x & 0x001f001f001f001f) + ($x >> 8 & 0x001f001f001f001f)
-        set $x = ($x & 0x0000003f0000003f) + ($x >>16 & 0x0000003f0000003f)
-        set $popcnt = ($x & 0x7f) + ($x >>32 & 0x7f)
-        set $index = $block->rank + $small_block_popcount + $popcnt
-      end
+      set $block_index = ($pos - $immediate_table_size) / 512
+      set $block = &$sd->succ_part[$block_index]
+      set $block_bit_index = ($pos - $immediate_table_size) % 512
+      set $small_block_index = $block_bit_index / 64
+      set $small_block_popcount = $small_block_index == 0 ? 0 : (((int)($block->small_block_ranks >> (($small_block_index - 1) * 9))) & 0x1ff)
+      set $x = $block->bits[$small_block_index] << (63 - $block_bit_index % 64)
+      set $x = ($x & 0x5555555555555555) + ($x >> 1 & 0x5555555555555555)
+      set $x = ($x & 0x3333333333333333) + ($x >> 2 & 0x3333333333333333)
+      set $x = ($x & 0x0707070707070707) + ($x >> 4 & 0x0707070707070707)
+      set $x = ($x & 0x001f001f001f001f) + ($x >> 8 & 0x001f001f001f001f)
+      set $x = ($x & 0x0000003f0000003f) + ($x >>16 & 0x0000003f0000003f)
+      set $popcnt = ($x & 0x7f) + ($x >>32 & 0x7f)
+      set $index = $block->rank + $small_block_popcount + $popcnt
     end
     printf "%d", $table[$index-1].line_no
   end

ast.c

@@ -359,7 +359,7 @@ ast_s_of(rb_execution_context_t *ec, VALUE module, VALUE body, VALUE keep_script
         rb_raise(rb_eRuntimeError, "cannot get AST for ISEQ compiled by prism");
     }
 
-    lines = ISEQ_BODY(iseq)->variable.script_lines;
+    lines = ISEQ_SCRIPT_LINES(iseq);
 
     VALUE path = rb_iseq_path(iseq);
     int e_option = RSTRING_LEN(path) == 2 && memcmp(RSTRING_PTR(path), "-e", 2) == 0;

compile.c

@@ -1015,7 +1015,8 @@ rb_iseq_translate_threaded_code(rb_iseq_t *iseq)
 VALUE *
 rb_iseq_original_iseq(const rb_iseq_t *iseq) /* cold path */
 {
-    VALUE *original_code = RUBY_ATOMIC_PTR_LOAD(ISEQ_BODY(iseq)->variable.original_iseq);
+    struct rb_iseq_variable *v = ISEQ_VARIABLE(iseq);
+    VALUE *original_code = v ? RUBY_ATOMIC_PTR_LOAD(v->original_iseq) : NULL;
 
     if (original_code) return original_code;
     original_code = ALLOC_N(VALUE, ISEQ_BODY(iseq)->iseq_size);
@@ -1037,7 +1038,8 @@ rb_iseq_original_iseq(const rb_iseq_t *iseq) /* cold path */
 
     /* Concurrent callers can each build a copy; publish only fully
      * translated code and keep the first one. */
-    VALUE *prev = ATOMIC_PTR_CAS(ISEQ_BODY(iseq)->variable.original_iseq,
+    v = rb_iseq_variable_ensure((rb_iseq_t *)iseq);
+    VALUE *prev = ATOMIC_PTR_CAS(v->original_iseq,
                                  NULL, original_code);
     if (prev) {
         SIZED_FREE_N(original_code, ISEQ_BODY(iseq)->iseq_size);
@@ -1530,7 +1532,7 @@ new_child_iseq(rb_iseq_t *iseq, const NODE *const node,
                                     line_no, parent,
                                     isolated_depth ? isolated_depth + 1 : 0,
                                     type, ISEQ_COMPILE_DATA(iseq)->option,
-                                    ISEQ_BODY(iseq)->variable.script_lines);
+                                    ISEQ_SCRIPT_LINES(iseq));
     debugs("[new_child_iseq]< ---------------------------------------\n");
     return ret_iseq;
 }
@@ -1700,10 +1702,8 @@ iseq_setup(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
     }
 
 #if VM_INSN_INFO_TABLE_IMPL == 2
-    if (ISEQ_BODY(iseq)->insns_info.succ_index_table == NULL) {
-        debugs("[compile step 7 (rb_iseq_insns_info_encode_positions)] \n");
-        rb_iseq_insns_info_encode_positions(iseq);
-    }
+    debugs("[compile step 7 (rb_iseq_insns_info_encode_positions)] \n");
+    rb_iseq_insns_info_encode_positions(iseq);
 #endif
 
     if (compile_debug > 1) {
@@ -2976,12 +2976,12 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
 
     /* get rid of memory leak when REALLOC failed */
     body->insns_info.body = insns_info;
-    body->insns_info.positions = positions;
+    body->insns_info.positions_or_succ_index_table.positions = positions;
 
     SIZED_REALLOC_N(insns_info, struct iseq_insn_info_entry, insns_info_index, insns_info_size);
     body->insns_info.body = insns_info;
     SIZED_REALLOC_N(positions, unsigned int, insns_info_index, positions_size);
-    body->insns_info.positions = positions;
+    body->insns_info.positions_or_succ_index_table.positions = positions;
     body->insns_info.size = insns_info_index;
 
     return COMPILE_OK;
@@ -9447,7 +9447,7 @@ compile_builtin_mandatory_only_method(rb_iseq_t *iseq, const NODE *node, const N
                            rb_iseq_path(iseq), rb_iseq_realpath(iseq),
                            nd_line(line_node), NULL, 0,
                            ISEQ_TYPE_METHOD, ISEQ_COMPILE_DATA(iseq)->option,
-                           ISEQ_BODY(iseq)->variable.script_lines);
+                           ISEQ_SCRIPT_LINES(iseq));
     RB_OBJ_WRITE(iseq, &ISEQ_BODY(iseq)->mandatory_only_iseq, (VALUE)mandatory_only_iseq);
 
     ALLOCV_END(idtmp);
@@ -13820,7 +13820,7 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq)
     ibf_dump_write_small_value(dump, mandatory_only_iseq_index);
     ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(ci_entries_offset));
     ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(outer_variables_offset));
-    ibf_dump_write_small_value(dump, body->variable.flip_count);
+    ibf_dump_write_small_value(dump, ISEQ_FLIP_CNT(iseq));
     ibf_dump_write_small_value(dump, body->local_table_size);
     ibf_dump_write_small_value(dump, body->ivc_size);
     ibf_dump_write_small_value(dump, body->icvarc_size);
@@ -14012,10 +14012,12 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset)
     load_body->ci_size = ci_size;
     load_body->insns_info.size = insns_info_size;
 
-    ISEQ_COVERAGE_SET(iseq, Qnil);
+    // variable is NULL from ZALLOC; only allocate if flip_count is non-zero.
     ISEQ_ORIGINAL_ISEQ_CLEAR(iseq);
-    load_body->variable.flip_count = variable_flip_count;
-    load_body->variable.script_lines = Qnil;
+    if (variable_flip_count) {
+        struct rb_iseq_variable *v = rb_iseq_variable_ensure(iseq);
+        v->flip_count = variable_flip_count;
+    }
 
     load_body->location.first_lineno = location_first_lineno;
     load_body->location.node_id = location_node_id;
@@ -14044,7 +14046,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset)
     load_body->param.keyword        = ibf_load_param_keyword(load, param_keyword_offset);
     load_body->param.flags.has_kw   = (param_flags >> 4) & 1;
     load_body->insns_info.body      = ibf_load_insns_info_body(load, insns_info_body_offset, insns_info_size);
-    load_body->insns_info.positions = ibf_load_insns_info_positions(load, insns_info_positions_offset, insns_info_size);
+    load_body->insns_info.positions_or_succ_index_table.positions = ibf_load_insns_info_positions(load, insns_info_positions_offset, insns_info_size);
     load_body->local_table          = ibf_load_local_table(load, local_table_offset, local_table_size);
     load_body->lvar_states          = ibf_load_lvar_states(load, lvar_states_offset, local_table_size, load_body->local_table);
     ibf_load_catch_table(load, catch_table_offset, catch_table_size, iseq);
@@ -15240,8 +15242,13 @@ rb_iseq_dup_with_independent_caches(const rb_iseq_t *src_root)
         struct rb_iseq_constant_body *cb = ISEQ_BODY(copy);
         if (!cb->local_iseq) RB_OBJ_WRITE(copy, &cb->local_iseq, sb->local_iseq);
         RB_OBJ_WRITE(copy, &cb->location.pathobj, sb->location.pathobj);
-        RB_OBJ_WRITE(copy, &cb->variable.script_lines, sb->variable.script_lines);
-        ISEQ_COVERAGE_SET(copy, ISEQ_COVERAGE(src_root));
+        VALUE sl = ISEQ_SCRIPT_LINES(src_root);
+        VALUE cov = ISEQ_COVERAGE(src_root);
+        if (!NIL_P(sl) || !NIL_P(cov)) {
+            struct rb_iseq_variable *v = rb_iseq_variable_ensure(copy);
+            RB_OBJ_WRITE(copy, &v->script_lines, sl);
+            RB_OBJ_WRITE(copy, &v->coverage, cov);
+        }
 
         if (i == 0) {
             RB_OBJ_WRITE(copy, &cb->parent_iseq, sb->parent_iseq);

ext/json/lib/json/common.rb

@@ -142,7 +142,56 @@ class JSONError < StandardError; end
 
   # This exception is raised if a parser error occurs.
   class ParserError < JSONError
-    attr_reader :line, :column
+    # Line number where the parser encountered an error.
+    # Is <tt>nil</tt> when raised by JSON::ResumableParser.
+    attr_reader :line
+
+    # Column number where the parser encountered an error.
+    # Is <tt>nil</tt> when raised by JSON::ResumableParser.
+    attr_reader :column
+
+    # Returns a best effort JSONPath string representing where in the document
+    # the parser encountered an error:
+    #
+    #   begin
+    #     JSON.parse('{"articles": [ { "title": invalid } ]}')
+    #   rescue JSON::ParserError => error
+    #     error.json_path # => "$.articles[0].title"
+    #   end
+    def json_path
+      return @json_path if String === @json_path
+
+      if Array === @json_path
+        path = build_json_path(@json_path)
+        @json_path = path unless frozen?
+        return path
+      end
+    end
+
+    private
+
+    def build_json_path(segments)
+      error = false
+      path = segments.filter_map do |segment|
+        next if error
+
+        case segment
+        when Integer
+          "[#{segment}]"
+        when String, Symbol
+          if segment.match?(/\A[a-zA-Z\$\_][a-zA-Z\$\_0-9]*\z/)
+            ".#{segment}"
+          else
+            segment = segment.to_s.gsub(/["\\]/, { '"' => '\\"', '\\' => '\\\\' })
+            %{["#{segment}"]}
+          end
+        else
+          error = true
+          nil
+        end
+      end.join
+      "$#{path}".freeze
+    end
   end
 
   # This exception is raised if the nesting of parsed data structures is too

ext/json/parser/parser.c

@@ -5,7 +5,7 @@
 static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8;
 static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string;
 
-static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column;
+static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column, i_at_json_path;
 #ifndef HAVE_RB_STR_TO_INTERNED_STR
 static ID i_uminus;
 #endif
@@ -651,11 +651,42 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta
     return rb_enc_sprintf(enc_utf8, format, ptr);
 }
 
+static VALUE json_path_new(JSON_ParserState *state, VALUE duplicate_key)
+{
+    VALUE path = rb_ary_new_capa(state->current_nesting);
+
+    json_frame_stack *frames = state->frames;
+    rvalue_stack *values = state->value_stack;
+
+    for (long depth = 1; depth < frames->head; depth++) {
+        json_frame *frame = &frames->ptr[depth];
+
+        bool innermost = depth == frames->head - 1;
+        long child_head = innermost ? values->head : frames->ptr[depth + 1].value_stack_head;
+        long count = child_head - frame->value_stack_head;
+
+        if (frame->type == JSON_FRAME_ARRAY) {
+            rb_ary_push(path, LONG2NUM(frame->phase == JSON_PHASE_ARRAY_COMMA ? count - 1 : count));
+        } else if (innermost && !UNDEF_P(duplicate_key)) {
+            rb_ary_push(path, duplicate_key);
+        } else if (count & 1) {
+            rb_ary_push(path, values->ptr[child_head - 1]);
+        } else if (frame->phase == JSON_PHASE_OBJECT_COMMA && count >= 2) {
+            rb_ary_push(path, values->ptr[child_head - 2]);
+        } else {
+            break;
+        }
+    }
+
+    return path;
+}
+
 static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos)
 {
     VALUE exc = rb_exc_new_str(eParserError, message);
     rb_ivar_set(exc, i_at_line, LONG2NUM(line));
     rb_ivar_set(exc, i_at_column, LONG2NUM(column));
+    rb_ivar_set(exc, i_at_json_path, json_path_new(state, Qundef));
     return exc;
 }
 
@@ -1199,14 +1230,17 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d
     );
 
     rb_str_concat(message, build_parse_error_message("", state));
+    VALUE exc;
     if (state->parser) { // line and columns can't be accurate in resumable
-        rb_exc_raise(parse_error_new(state, message, 0, 0, false));
+        exc = parse_error_new(state, message, 0, 0, false);
     } else {
         long line, column;
         cursor_position(state, &line, &column);
         rb_str_catf(message, " at line %ld column %ld", line, column);
-        rb_exc_raise(parse_error_new(state, message, line, column, false));
+        exc = parse_error_new(state, message, line, column, false);
     }
+    rb_ivar_set(exc, i_at_json_path, json_path_new(state, duplicate_key));
+    rb_exc_raise(exc);
 }
 
 NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs)
@@ -2122,6 +2156,12 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
     // the rvalue stack.
     VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef;
 
+    if (complete) {
+        json_ensure_eof(state, config);
+    } else {
+        raise_eos_error("unexpected end of input", state);
+    }
+
     // This may be skipped in case of exception, but
     // it won't cause a leak.
     rvalue_stack_eagerly_release(value_stack_handle);
@@ -2130,12 +2170,6 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
     RB_GC_GUARD(frame_stack_handle);
     RB_GC_GUARD(Vsource);
 
-    if (complete) {
-        json_ensure_eof(state, config);
-    } else {
-        raise_eos_error("unexpected end of input", state);
-    }
-
     return result;
 }
 
@@ -2871,6 +2905,7 @@ void Init_parser(void)
     i_encode = rb_intern("encode");
     i_at_line = rb_intern("@line");
     i_at_column = rb_intern("@column");
+    i_at_json_path = rb_intern("@json_path");
 
     binary_encindex = rb_ascii8bit_encindex();
     utf8_encindex = rb_utf8_encindex();

include/ruby/io/buffer.h

@@ -29,6 +29,9 @@ RUBY_EXTERN VALUE rb_cIOBuffer;
 // The operating system page size.
 RUBY_EXTERN size_t RUBY_IO_BUFFER_PAGE_SIZE;
 
+// The alignment required for file mapping offsets.
+RUBY_EXTERN size_t RUBY_IO_BUFFER_MAP_ALIGNMENT;
+
 // The default buffer size, usually a (small) multiple of the page size.
 // Can be overridden by the RUBY_IO_BUFFER_DEFAULT_SIZE environment variable.
 RUBY_EXTERN size_t RUBY_IO_BUFFER_DEFAULT_SIZE;