Implement negative line numbers

docs/serialization.md

@@ -9,24 +9,28 @@ The syntax tree still requires a copy of the original source, as for the most pa
 
 Let us define some simple types for readability.
 
-### varint
+### varuint
 
-A variable-length integer with the value fitting in `uint32_t` using between 1 and 5 bytes, using the [LEB128](https://en.wikipedia.org/wiki/LEB128) encoding.
+A variable-length unsigned integer with the value fitting in `uint32_t` using between 1 and 5 bytes, using the [LEB128](https://en.wikipedia.org/wiki/LEB128) encoding.
 This drastically cuts down on the size of the serialized string, especially when the source file is large.
 
+### varsint
+
+A variable-length signed integer with the value fitting in `int32_t` using between 1 and 5 bytes, using [ZigZag encoding](https://protobuf.dev/programming-guides/encoding/#signed-ints) into [LEB128].
+
 ### string
 
 | # bytes | field |
 | --- | --- |
-| varint | the length of the string in bytes |
+| varuint | the length of the string in bytes |
 | ... | the string bytes |
 
 ### location
 
 | # bytes | field |
 | --- | --- |
-| varint | byte offset into the source string where this location begins |
-| varint | length of the location in bytes in the source string |
+| varuint | byte offset into the source string where this location begins |
+| varuint | length of the location in bytes in the source string |
 
 ### comment
 
@@ -71,18 +75,18 @@ The header is structured like the following table:
 | `1` | patch version number |
 | `1` | 1 indicates only semantics fields were serialized, 0 indicates all fields were serialized (including location fields) |
 | string | the encoding name |
-| varint | the start line |
-| varint | number of comments |
+| varsint | the start line |
+| varuint | number of comments |
 | comment* | comments |
-| varint | number of magic comments |
+| varuint | number of magic comments |
 | magic comment* | magic comments |
 | location? | the optional location of the `__END__` keyword and its contents |
-| varint | number of errors |
+| varuint | number of errors |
 | diagnostic* | errors |
-| varint | number of warnings |
+| varuint | number of warnings |
 | diagnostic* | warnings |
 | `4` | content pool offset |
-| varint | content pool size |
+| varuint | content pool size |
 
 After the header comes the body of the serialized string.
 The body consists of a sequence of nodes that is built using a prefix traversal order of the syntax tree.
@@ -159,7 +163,7 @@ serialize(const uint8_t *source, size_t length) {
 }
 ```
 
-The final argument to `pm_serialize_parse` is an optional string that controls the options to the parse function. This includes all of the normal options that could be passed to `pm_parser_init` through a `pm_options_t` struct, but serialized as a string to make it easier for callers through FFI. Note that no `varint` are used here to make it easier to produce the data for the caller, and also serialized size is less important here. The format of the data is structured as follows:
+The final argument to `pm_serialize_parse` is an optional string that controls the options to the parse function. This includes all of the normal options that could be passed to `pm_parser_init` through a `pm_options_t` struct, but serialized as a string to make it easier for callers through FFI. Note that no `varuint` are used here to make it easier to produce the data for the caller, and also serialized size is less important here. The format of the data is structured as follows:
 
 | # bytes | field                      |
 | ------- | -------------------------- |

ext/prism/extension.c

@@ -126,7 +126,7 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
     } else if (key_id == rb_option_id_encoding) {
         if (!NIL_P(value)) pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value)));
     } else if (key_id == rb_option_id_line) {
-        if (!NIL_P(value)) pm_options_line_set(options, NUM2UINT(value));
+        if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value));
     } else if (key_id == rb_option_id_frozen_string_literal) {
         if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, value == Qtrue);
     } else if (key_id == rb_option_id_verbose) {
@@ -166,6 +166,7 @@ build_options(VALUE argument) {
  */
 static void
 extract_options(pm_options_t *options, VALUE filepath, VALUE keywords) {
+    options->line = 1; // default
     if (!NIL_P(keywords)) {
         struct build_options_data data = { .options = options, .keywords = keywords };
         struct build_options_data *argument = &data;

include/prism/options.h

@@ -35,7 +35,7 @@ typedef struct {
      * The line within the file that the parse starts on. This value is
      * 0-indexed.
      */
-    uint32_t line;
+    int32_t line;
 
     /**
      * The name of the encoding that the source file is in. Note that this must
@@ -80,7 +80,7 @@ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, cons
  * @param options The options struct to set the line on.
  * @param line The line to set.
  */
-PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, uint32_t line);
+PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t line);
 
 /**
  * Set the encoding option on the given options struct.

include/prism/parser.h

@@ -661,7 +661,7 @@ struct pm_parser {
      * The line number at the start of the parse. This will be used to offset
      * the line numbers of all of the locations.
      */
-    uint32_t start_line;
+    int32_t start_line;
 
     /** Whether or not we're at the beginning of a command. */
     bool command_start;

include/prism/util/pm_buffer.h

@@ -118,7 +118,15 @@ void pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value);
  * @param buffer The buffer to append to.
  * @param value The integer to append.
  */
-void pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value);
+void pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value);
+
+/**
+ * Append a 32-bit signed integer to the buffer as a variable-length integer.
+ *
+ * @param buffer The buffer to append to.
+ * @param value The integer to append.
+ */
+void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
 
 /**
  * Concatenate one buffer onto another.

src/options.c

@@ -20,7 +20,7 @@ pm_options_encoding_set(pm_options_t *options, const char *encoding) {
  * Set the line option on the given options struct.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_options_line_set(pm_options_t *options, uint32_t line) {
+pm_options_line_set(pm_options_t *options, int32_t line) {
     options->line = line;
 }
 
@@ -114,6 +114,22 @@ pm_options_read_u32(const char *data) {
     }
 }
 
+/**
+ * Read a 32-bit signed integer from a pointer. This function is used to read
+ * the options that are passed into the parser from the Ruby implementation. It
+ * handles aligned and unaligned reads.
+ */
+static int32_t
+pm_options_read_s32(const char *data) {
+    if (((uintptr_t) data) % sizeof(int32_t) == 0) {
+        return *((int32_t *) data);
+    } else {
+        int32_t value;
+        memcpy(&value, data, sizeof(int32_t));
+        return value;
+    }
+}
+
 /**
  * Deserialize an options struct from the given binary string. This is used to
  * pass options to the parser from an FFI call so that consumers of the library
@@ -123,6 +139,9 @@ pm_options_read_u32(const char *data) {
  */
 void
 pm_options_read(pm_options_t *options, const char *data) {
+    options->line = 1; // default
+    if (data == NULL) return;
+
     uint32_t filepath_length = pm_options_read_u32(data);
     data += 4;
 
@@ -131,7 +150,7 @@ pm_options_read(pm_options_t *options, const char *data) {
         data += filepath_length;
     }
 
-    options->line = pm_options_read_u32(data);
+    options->line = pm_options_read_s32(data);
     data += 4;
 
     uint32_t encoding_length = pm_options_read_u32(data);

src/prism.c

@@ -17067,9 +17067,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm
         parser->filepath_string = options->filepath;
 
         // line option
-        if (options->line > 0) {
-            parser->start_line = options->line;
-        }
+        parser->start_line = options->line;
 
         // encoding option
         size_t encoding_length = pm_string_length(&options->encoding);
@@ -17238,7 +17236,7 @@ pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
 PRISM_EXPORTED_FUNCTION void
 pm_serialize_parse(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) {
     pm_options_t options = { 0 };
-    if (data != NULL) pm_options_read(&options, data);
+    pm_options_read(&options, data);
 
     pm_parser_t parser;
     pm_parser_init(&parser, source, size, &options);
@@ -17260,15 +17258,15 @@ pm_serialize_parse(pm_buffer_t *buffer, const uint8_t *source, size_t size, cons
 PRISM_EXPORTED_FUNCTION void
 pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) {
     pm_options_t options = { 0 };
-    if (data != NULL) pm_options_read(&options, data);
+    pm_options_read(&options, data);
 
     pm_parser_t parser;
     pm_parser_init(&parser, source, size, &options);
 
     pm_node_t *node = pm_parse(&parser);
     pm_serialize_header(buffer);
     pm_serialize_encoding(&parser.encoding, buffer);
-    pm_buffer_append_varint(buffer, parser.start_line);
+    pm_buffer_append_varsint(buffer, parser.start_line);
     pm_serialize_comment_list(&parser, &parser.comment_list, buffer);
 
     pm_node_destroy(&parser, node);

src/util/pm_buffer.c

@@ -138,7 +138,7 @@ pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value) {
  * Append a 32-bit unsigned integer to the buffer as a variable-length integer.
  */
 void
-pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value) {
+pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value) {
     if (value < 128) {
         pm_buffer_append_byte(buffer, (uint8_t) value);
     } else {
@@ -151,6 +151,15 @@ pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value) {
     }
 }
 
+/**
+ * Append a 32-bit signed integer to the buffer as a variable-length integer.
+ */
+void
+pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value) {
+    uint32_t unsigned_int = ((uint32_t)(value) << 1) ^ ((uint32_t)(value >> 31));
+    pm_buffer_append_varuint(buffer, unsigned_int);
+}
+
 /**
  * Concatenate one buffer onto another.
  */

templates/ext/prism/api_node.c.erb

@@ -46,7 +46,7 @@ pm_source_new(pm_parser_t *parser, rb_encoding *encoding) {
         rb_ary_push(offsets, INT2FIX(parser->newline_list.offsets[index]));
     }
 
-    VALUE source_argv[] = { source, ULONG2NUM(parser->start_line), offsets };
+    VALUE source_argv[] = { source, LONG2NUM(parser->start_line), offsets };
     return rb_class_new_instance(3, source_argv, rb_cPrismSource);
 }