From 577c302d413953b6c0fd5d4ea44eb0099756a1e4 Mon Sep 17 00:00:00 2001 From: Mikhail Tavarez Date: Tue, 9 Jul 2024 12:04:01 -0500 Subject: [PATCH] updating for nightly --- gojo/bufio/__init__.mojo | 2 +- gojo/bufio/bufio.mojo | 106 +++++++++++------------------ gojo/bufio/scan.mojo | 68 +++++++++++-------- gojo/builtins/__init__.mojo | 6 +- gojo/builtins/attributes.mojo | 110 ++++++++++++++++-------------- gojo/builtins/errors.mojo | 2 +- gojo/bytes/buffer.mojo | 117 +++++++++++++------------------- gojo/bytes/reader.mojo | 33 +++------ gojo/fmt/fmt.mojo | 68 +++---------------- gojo/io/__init__.mojo | 8 +-- gojo/io/file.mojo | 25 +++---- gojo/io/io.mojo | 18 ++--- gojo/io/std.mojo | 3 +- gojo/net/fd.mojo | 10 +-- gojo/net/ip.mojo | 16 ++--- gojo/net/socket.mojo | 25 +------ gojo/net/tcp.mojo | 2 +- gojo/net/udp.mojo | 2 +- gojo/strings/builder.mojo | 25 ++----- gojo/strings/reader.mojo | 28 +++----- gojo/unicode/__init__.mojo | 2 +- gojo/unicode/utf8/__init__.mojo | 1 - gojo/unicode/utf8/runes.mojo | 9 +-- gojo/unicode/utf8/string.mojo | 109 ----------------------------- gojo/unicode/utf8/width.mojo | 3 +- tests/test_bufio.mojo | 20 +++--- tests/test_bufio_scanner.mojo | 57 ++++++++-------- tests/test_builtins_bytes.mojo | 14 +--- tests/test_bytes_buffer.mojo | 24 +++---- tests/test_fmt.mojo | 3 - tests/test_get_addr.mojo | 2 +- tests/test_std.mojo | 4 +- tests/test_strings_reader.mojo | 14 ++-- tests/test_unicode_string.mojo | 23 ------- tests/test_unicode_width.mojo | 8 +-- 35 files changed, 328 insertions(+), 639 deletions(-) delete mode 100644 gojo/unicode/utf8/string.mojo delete mode 100644 tests/test_unicode_string.mojo diff --git a/gojo/bufio/__init__.mojo b/gojo/bufio/__init__.mojo index 2255063..2028d55 100644 --- a/gojo/bufio/__init__.mojo +++ b/gojo/bufio/__init__.mojo @@ -1,2 +1,2 @@ from .bufio import Reader, Writer, ReadWriter, new_writer, new_reader, new_read_writer -from .scan import Scanner, scan_words, scan_bytes, scan_lines, scan_runes +from .scan import Scanner, scan_words, scan_bytes, scan_lines diff --git a/gojo/bufio/bufio.mojo b/gojo/bufio/bufio.mojo index ffe8c79..5001093 100644 --- a/gojo/bufio/bufio.mojo +++ b/gojo/bufio/bufio.mojo @@ -28,7 +28,6 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, var last_rune_size: Int # size of last rune read for unread_rune; -1 means invalid var err: Error - @always_inline fn __init__( inout self, owned reader: R, @@ -49,7 +48,6 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, self.last_rune_size = last_rune_size self.err = Error() - @always_inline fn __moveinit__(inout self, owned existing: Self): self.buf = InlineList[UInt8, size]() for element in existing.buf: @@ -63,7 +61,6 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, self.err = existing.err^ # size returns the size of the underlying buffer in bytes. - @always_inline fn __len__(self) -> Int: return len(self.buf) @@ -84,12 +81,10 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, # self.reset(self.buf, r) - @always_inline - fn as_bytes_slice(inout self) -> Span[UInt8, True, __lifetime_of(self)]: - """Returns the internal buffer data as a Span[UInt8].""" - return Span[UInt8, True, __lifetime_of(self)](array=Reference(self.buf._array)) + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](array=self.buf._array) - @always_inline fn reset(inout self, buf: InlineList[UInt8, size], owned reader: R): """Discards any buffered data, resets all state, and switches the buffered reader to read from r. @@ -146,7 +141,6 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, self.err = Error(str(io.ERR_NO_PROGRESS)) - @always_inline fn read_error(inout self) -> Error: if not self.err: return Error() @@ -155,7 +149,7 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, self.err = Error() return err - fn peek(self: Reference[Self, True], number_of_bytes: Int) -> (Span[UInt8, self.is_mutable, self.lifetime], Error): + fn peek(inout self, number_of_bytes: Int) -> (Span[UInt8, __lifetime_of(self)], Error): """Returns the next n bytes without advancing the reader. The bytes stop being valid at the next read call. If Peek returns fewer than n bytes, it also returns an error explaining why the read is short. The error is @@ -168,29 +162,27 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, number_of_bytes: The number of bytes to peek. """ if number_of_bytes < 0: - return self[].as_bytes_slice()[0:0], Error(ERR_NEGATIVE_COUNT) + return self.as_bytes_slice()[0:0], Error(ERR_NEGATIVE_COUNT) - self[].last_byte = -1 - self[].last_rune_size = -1 + self.last_byte = -1 + self.last_rune_size = -1 - while ( - self[].write_pos - self[].read_pos < number_of_bytes and self[].write_pos - self[].read_pos < io.BUFFER_SIZE - ): - self[].fill() # self.write_pos-self.read_pos < self.capacity => buffer is not full + while self.write_pos - self.read_pos < number_of_bytes and self.write_pos - self.read_pos < io.BUFFER_SIZE: + self.fill() # self.write_pos-self.read_pos < self.capacity => buffer is not full if number_of_bytes > io.BUFFER_SIZE: - return self[].as_bytes_slice()[self[].read_pos : self[].write_pos], Error(ERR_BUFFER_FULL) + return self.as_bytes_slice()[self.read_pos : self.write_pos], Error(ERR_BUFFER_FULL) # 0 <= n <= io.BUFFER_SIZE var err = Error() - var available_space = self[].write_pos - self[].read_pos + var available_space = self.write_pos - self.read_pos if available_space < number_of_bytes: # not enough data in buffer - err = self[].read_error() + err = self.read_error() if not err: err = Error(ERR_BUFFER_FULL) - return self[].as_bytes_slice()[self[].read_pos : self[].read_pos + number_of_bytes], err + return self.as_bytes_slice()[self.read_pos : self.read_pos + number_of_bytes], err fn discard(inout self, number_of_bytes: Int) -> (Int, Error): """Discard skips the next n bytes, returning the number of bytes discarded. @@ -223,7 +215,7 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, if remain == 0: return number_of_bytes, Error() - fn _read(inout self, inout dest: Span[UInt8, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Reads data into dest. It returns the number of bytes read into dest. The bytes are taken from at most one Read on the underlying [Reader], @@ -273,9 +265,9 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, var source = self.as_bytes_slice()[self.read_pos : self.write_pos] bytes_read = 0 var start = len(dest) - + var target = dest.unsafe_ptr() for i in range(len(source)): - dest[i + start] = source[i] + target[i + start] = source[i] bytes_read += 1 dest._len += bytes_read self.read_pos += bytes_read @@ -283,7 +275,6 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, self.last_rune_size = -1 return bytes_read, Error() - @always_inline fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): """Reads data into dest. It returns the number of bytes read into dest. @@ -302,7 +293,6 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, return bytes_read, err - @always_inline fn read_byte(inout self) -> (UInt8, Error): """Reads and returns a single byte from the internal buffer. If no byte is available, returns an error.""" self.last_rune_size = -1 @@ -316,7 +306,6 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, self.last_byte = int(c) return c, Error() - @always_inline fn unread_byte(inout self) -> Error: """Unreads the last byte. Only the most recently read byte can be unread. @@ -372,7 +361,6 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, # self.last_rune_size = -1 # return nil - @always_inline fn buffered(self) -> Int: """Returns the number of bytes that can be read from the current buffer. @@ -381,7 +369,7 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, """ return self.write_pos - self.read_pos - fn read_slice(self: Reference[Self, True], delim: UInt8) -> (Span[UInt8, self.is_mutable, self.lifetime], Error): + fn read_slice(inout self, delim: UInt8) -> (Span[UInt8, __lifetime_of(self)], Error): """Reads until the first occurrence of delim in the input, returning a slice pointing at the bytes in the buffer. It includes the first occurrence of the delimiter. The bytes stop being valid at the next read. @@ -401,38 +389,38 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, """ var err = Error() var s = 0 # search start index - var line: Span[UInt8, self.is_mutable, self.lifetime] + var line: Span[UInt8, __lifetime_of(self)] while True: # Search buffer. - var i = index_byte(self[].as_bytes_slice()[self[].read_pos + s : self[].write_pos], delim) + var i = index_byte(self.as_bytes_slice()[self.read_pos + s : self.write_pos], delim) if i >= 0: i += s - line = self[].as_bytes_slice()[self[].read_pos : self[].read_pos + i + 1] - self[].read_pos += i + 1 + line = self.as_bytes_slice()[self.read_pos : self.read_pos + i + 1] + self.read_pos += i + 1 break # Pending error? - if self[].err: - line = self[].as_bytes_slice()[self[].read_pos : self[].write_pos] - self[].read_pos = self[].write_pos - err = self[].read_error() + if self.err: + line = self.as_bytes_slice()[self.read_pos : self.write_pos] + self.read_pos = self.write_pos + err = self.read_error() break # Buffer full? - if self[].buffered() >= io.BUFFER_SIZE: - self[].read_pos = self[].write_pos - line = self[].as_bytes_slice() + if self.buffered() >= io.BUFFER_SIZE: + self.read_pos = self.write_pos + line = self.as_bytes_slice() err = Error(ERR_BUFFER_FULL) break - s = self[].write_pos - self[].read_pos # do not rescan area we scanned before - self[].fill() # buffer is not full + s = self.write_pos - self.read_pos # do not rescan area we scanned before + self.fill() # buffer is not full # Handle last byte, if any. var i = len(line) - 1 if i >= 0: - self[].last_byte = int(line[i]) - self[].last_rune_size = -1 + self.last_byte = int(line[i]) + self.last_rune_size = -1 return line, err @@ -454,7 +442,7 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, (possibly a character belonging to the line end) even if that byte is not part of the line returned by read_line. """ - var line: Span[UInt8, True, __lifetime_of(self)] + var line: Span[UInt8, __lifetime_of(self)] var err: Error line, err = self.read_slice(ord("\n")) @@ -483,9 +471,7 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, return List[UInt8](line), False - fn collect_fragments( - inout self, delim: UInt8 - ) -> (List[List[UInt8]], Span[UInt8, True, __lifetime_of(self)], Int, Error): + fn collect_fragments(inout self, delim: UInt8) -> (List[List[UInt8]], Span[UInt8, __lifetime_of(self)], Int, Error): """Reads until the first occurrence of delim in the input. It returns (slice of full buffers, remaining bytes before delim, total number of bytes in the combined first two elements, error). @@ -497,7 +483,7 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, var err = Error() var full_buffers = List[List[UInt8]]() var total_len = 0 - var frag: Span[UInt8, True, __lifetime_of(self)] + var frag: Span[UInt8, __lifetime_of(self)] while True: frag, err = self.read_slice(delim) if not err: @@ -532,7 +518,7 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, The List[UInt8] from the internal buffer. """ var full: List[List[UInt8]] - var frag: Span[UInt8, True, __lifetime_of(self)] + var frag: Span[UInt8, __lifetime_of(self)] var n: Int var err: Error full, frag, n, err = self.collect_fragments(delim) @@ -566,7 +552,7 @@ struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, The String from the internal buffer. """ var full: List[List[UInt8]] - var frag: Span[UInt8, True, __lifetime_of(self)] + var frag: Span[UInt8, __lifetime_of(self)] var n: Int var err: Error full, frag, n, err = self.collect_fragments(delim) @@ -680,7 +666,6 @@ struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( var writer: W var err: Error - @always_inline fn __init__( inout self, owned writer: W, @@ -694,7 +679,6 @@ struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( self.writer = writer^ self.err = Error() - @always_inline fn __moveinit__(inout self, owned existing: Self): self.buf = InlineList[UInt8, size]() for element in existing.buf: @@ -703,17 +687,14 @@ struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( self.writer = existing.writer^ self.err = existing.err^ - @always_inline fn __len__(self) -> Int: """Returns the size of the underlying buffer in bytes.""" return len(self.buf) - @always_inline - fn as_bytes_slice(inout self) -> Span[UInt8, True, __lifetime_of(self)]: - """Returns the internal buffer data as a Span[UInt8].""" - return Span[UInt8, True, __lifetime_of(self)](array=Reference(self.buf._array)) + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](array=self.buf._array) - @always_inline fn reset(inout self, owned writer: W): """Discards any unflushed buffered data, clears any error, and resets b to write its output to w. @@ -772,12 +753,10 @@ struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( self.bytes_written = 0 return err - @always_inline fn available(self) -> Int: """Returns how many bytes are unused in the buffer.""" return self.buf.capacity - len(self.buf) - @always_inline fn buffered(self) -> Int: """Returns the number of bytes that have been written into the current buffer. @@ -838,7 +817,6 @@ struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( total_bytes_written += n return total_bytes_written, err - @always_inline fn write(inout self, src: List[UInt8]) -> (Int, Error): """ Appends a byte List to the builder buffer. @@ -854,7 +832,6 @@ struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( return bytes_read, err - @always_inline fn write_byte(inout self, src: UInt8) -> (Int, Error): """Writes a single byte to the internal buffer. @@ -901,7 +878,6 @@ struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( # self.bytes_written += size # return size, nil - @always_inline fn write_string(inout self, src: String) -> (Int, Error): """Writes a string to the internal buffer. It returns the number of bytes written. @@ -944,7 +920,7 @@ struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( while nr < MAX_CONSECUTIVE_EMPTY_READS: # Read into remaining unused space in the buffer. var buf = self.as_bytes_slice()[self.bytes_written : len(self.buf)] - bytes_read, err = reader._read(buf, len(buf)) + bytes_read, err = reader._read(buf, self.bytes_written - len(self.buf)) if bytes_read != 0 or err: break diff --git a/gojo/bufio/scan.mojo b/gojo/bufio/scan.mojo index 64a7cd3..e538d4d 100644 --- a/gojo/bufio/scan.mojo +++ b/gojo/bufio/scan.mojo @@ -58,10 +58,23 @@ struct Scanner[R: io.Reader, split: SplitFunction = scan_lines](): # The functi self.done = done self.err = Error() - @always_inline - fn as_bytes_slice(self: Reference[Self]) -> Span[UInt8, self.is_mutable, self.lifetime]: - """Returns the internal buffer data as a Span[UInt8].""" - return Span[UInt8, self.is_mutable, self.lifetime](self[].buf) + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](self.buf) + + fn current_token_as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the most recent token generated by a call to [Scanner.Scan]. + The underlying array may point to data that will be overwritten + by a subsequent call to Scan. It does no allocation. + """ + return Span[UInt8, __lifetime_of(self)](self.token) + + fn current_token_as_string_slice(ref [_]self) -> StringSlice[__lifetime_of(self)]: + """Returns the most recent token generated by a call to [Scanner.Scan]. + The underlying array may point to data that will be overwritten + by a subsequent call to Scan. It does no allocation. + """ + return StringSlice[__lifetime_of(self)](unsafe_from_utf8_ptr=self.token.unsafe_ptr(), len=len(self.token)) fn current_token_as_bytes(self) -> List[UInt8]: """Returns the most recent token generated by a call to [Scanner.Scan]. @@ -73,9 +86,7 @@ struct Scanner[R: io.Reader, split: SplitFunction = scan_lines](): # The functi fn current_token(self) -> String: """Returns the most recent token generated by a call to [Scanner.Scan] as a newly allocated string holding its bytes.""" - var copy = self.token - copy.append(0) - return String(copy^) + return self.current_token_as_string_slice() fn scan(inout self) -> Bool: """Advances the [Scanner] to the next token, which will then be @@ -172,11 +183,10 @@ struct Scanner[R: io.Reader, split: SplitFunction = scan_lines](): # The functi var loop = 0 while True: var buf = self.as_bytes_slice()[self.end :] - # Catch any reader errors and set the internal error field to that err instead of bubbling it up. var bytes_read: Int var err: Error - bytes_read, err = self.reader._read(buf, self.buf.capacity - self.end) + bytes_read, err = self.reader._read(buf, len(buf)) if bytes_read < 0 or len(buf) - self.end < bytes_read: self.set_err(ERR_BAD_READ_COUNT) break @@ -309,31 +319,31 @@ fn scan_bytes(data: Span[UInt8], at_eof: Bool) -> (Int, List[UInt8], Error): return 1, List[UInt8](data[0:1]), Error() -fn scan_runes(data: Span[UInt8], at_eof: Bool) -> (Int, List[UInt8], Error): - """Returns each UTF-8-encoded rune as a token. +# TODO: Fix this and uncomment it. +# fn scan_runes(data: Span[UInt8], at_eof: Bool) -> (Int, List[UInt8], Error): +# """Returns each UTF-8 encoded rune as a token. - Args: - data: The data to split. - at_eof: Whether the data is at the end of the file. +# Args: +# data: The data to split. +# at_eof: Whether the data is at the end of the file. - Returns: - The number of bytes to advance the input, token in bytes, and an error if one occurred. - """ - if at_eof and len(data) == 0: - return 0, List[UInt8](), Error() +# Returns: +# The number of bytes to advance the input, token in bytes, and an error if one occurred. +# """ +# if at_eof and len(data) == 0: +# return 0, List[UInt8](), Error() - # Number of bytes of the current character - var char_length = int( - (DTypePointer[DType.uint8](data.unsafe_ptr()).load() >> 7 == 0).cast[DType.uint8]() * 1 - + countl_zero(~DTypePointer[DType.uint8](data.unsafe_ptr()).load()) - ) +# # Number of bytes of the current character +# var lhs = (SIMD[size=1].load(DTypePointer[DType.uint8](data.unsafe_ptr())) >> 7 == 0 * 1).cast[DType.uint8]() +# var rhs = countl_zero(~SIMD[size=1].load(DTypePointer[DType.uint8](data.unsafe_ptr()))) +# var char_length = int(lhs + rhs) - # Copy N bytes into new pointer and construct List. - var sp = UnsafePointer[UInt8].alloc(char_length) - memcpy(sp, data.unsafe_ptr(), char_length) - var result = List[UInt8](unsafe_pointer=sp, size=char_length, capacity=char_length) +# # Copy N bytes into new pointer and construct List. +# var sp = UnsafePointer[UInt8].alloc(char_length) +# memcpy(sp, data.unsafe_ptr(), char_length) +# var result = List[UInt8](unsafe_pointer=sp, size=char_length, capacity=char_length) - return char_length, result, Error() +# return char_length, result, Error() fn drop_carriage_return(data: Span[UInt8]) -> List[UInt8]: diff --git a/gojo/builtins/__init__.mojo b/gojo/builtins/__init__.mojo index 0827829..1d2e998 100644 --- a/gojo/builtins/__init__.mojo +++ b/gojo/builtins/__init__.mojo @@ -1,5 +1,3 @@ -from .bytes import Byte, index_byte, has_suffix, has_prefix, to_string -from .attributes import cap, copy +from .bytes import index_byte, has_suffix, has_prefix, to_string +from .attributes import copy from .errors import exit, panic - -alias Rune = Int32 diff --git a/gojo/builtins/attributes.mojo b/gojo/builtins/attributes.mojo index 7ce0b15..5202fff 100644 --- a/gojo/builtins/attributes.mojo +++ b/gojo/builtins/attributes.mojo @@ -25,64 +25,82 @@ fn copy[T: CollectionElement](inout target: List[T], source: List[T], start: Int return count -fn copy[T: CollectionElement](inout target_span: Span[T, True], source_span: Span[T], start: Int = 0) -> Int: - """Copies the contents of source into target at the same index. Returns the number of bytes copied. - Added a start parameter to specify the index to start copying into. +# fn copy[T: CollectionElement](inout target_span: Span[T], source_span: Span[T], start: Int = 0) -> Int: +# """Copies the contents of source into target at the same index. Returns the number of bytes copied. +# Added a start parameter to specify the index to start copying into. - Args: - target_span: The buffer to copy into. - source_span: The buffer to copy from. - start: The index to start copying into. +# Args: +# target_span: The buffer to copy into. +# source_span: The buffer to copy from. +# start: The index to start copying into. - Returns: - The number of bytes copied. - """ - var count = 0 +# Returns: +# The number of bytes copied. +# """ +# var count = 0 - for i in range(len(source_span)): - target_span[i + start] = source_span[i] - count += 1 +# for i in range(len(source_span)): +# target_span[i + start] = source_span[i] +# count += 1 - target_span._len += count - return count +# target_span._len += count +# return count -fn copy[T: CollectionElement](inout target_span: Span[T, True], source: InlineList[T], start: Int = 0) -> Int: - """Copies the contents of source into target at the same index. Returns the number of bytes copied. - Added a start parameter to specify the index to start copying into. +# fn copy[T: CollectionElementNew](inout target_span: Span[T], source: InlineList[T], start: Int = 0) -> Int: +# """Copies the contents of source into target at the same index. Returns the number of bytes copied. +# Added a start parameter to specify the index to start copying into. - Args: - target_span: The buffer to copy into. - source: The buffer to copy from. - start: The index to start copying into. +# Args: +# target_span: The buffer to copy into. +# source: The buffer to copy from. +# start: The index to start copying into. - Returns: - The number of bytes copied. - """ - var count = 0 +# Returns: +# The number of bytes copied. +# """ +# var count = 0 - for i in range(len(source)): - target_span[i + start] = source[i] - count += 1 +# for i in range(len(source)): +# target_span[i + start] = source[i] +# count += 1 - target_span._len += count - return count +# target_span._len += count +# return count + + +# fn copy[T: CollectionElementNew, T2: CollectionElement](inout list: InlineList[T], source: Span[T2], start: Int = 0) -> Int: +# """Copies the contents of source into target at the same index. Returns the number of bytes copied. +# Added a start parameter to specify the index to start copying into. + +# Args: +# list: The buffer to copy into. +# source: The buffer to copy from. +# start: The index to start copying into. +# Returns: +# The number of bytes copied. +# """ +# var count = 0 -fn test(inout dest: List[UInt8]): - var source = List[UInt8](1, 2, 3) - var target = Span[UInt8](dest) +# for i in range(len(source)): +# if i + start > len(list): +# list[i + start] = source[i] +# else: +# list.append(source[i]) +# count += 1 - _ = copy(target, Span(source), start=0) +# return count -fn copy[T: CollectionElement](inout list: InlineList[T], source: Span[T], start: Int = 0) -> Int: +fn copy(target: UnsafePointer[UInt8], source: UnsafePointer[UInt8], source_length: Int, start: Int = 0) -> Int: """Copies the contents of source into target at the same index. Returns the number of bytes copied. Added a start parameter to specify the index to start copying into. Args: - list: The buffer to copy into. + target: The buffer to copy into. source: The buffer to copy from. + source_length: The length of the source buffer. start: The index to start copying into. Returns: @@ -90,11 +108,8 @@ fn copy[T: CollectionElement](inout list: InlineList[T], source: Span[T], start: """ var count = 0 - for i in range(len(source)): - if i + start > len(list): - list[i + start] = source[i] - else: - list.append(source[i]) + for i in range(source_length): + target[i + start] = source[i] count += 1 return count @@ -130,12 +145,3 @@ fn copy( count += 1 return count - - -fn cap[T: CollectionElement](iterable: List[T]) -> Int: - """Returns the capacity of the List. - - Args: - iterable: The List to get the capacity of. - """ - return iterable.capacity diff --git a/gojo/builtins/errors.mojo b/gojo/builtins/errors.mojo index 1c1a7ba..0001ae7 100644 --- a/gojo/builtins/errors.mojo +++ b/gojo/builtins/errors.mojo @@ -8,5 +8,5 @@ fn panic[T: Stringable](message: T, code: Int = 1): message: The message to panic with. code: The exit code to panic with. """ - print("panic:", message) + print("panic:", str(message)) exit(code) diff --git a/gojo/bytes/buffer.mojo b/gojo/bytes/buffer.mojo index 668f9fe..8ef8d2e 100644 --- a/gojo/bytes/buffer.mojo +++ b/gojo/bytes/buffer.mojo @@ -1,5 +1,5 @@ import ..io -from ..builtins import copy, Byte, panic, index_byte +from ..builtins import copy, panic, index_byte from algorithm.memory import parallel_memcpy @@ -51,7 +51,6 @@ struct Buffer( var offset: Int # read at &buf[off], write at &buf[len(buf)] var last_read: ReadOp # last read operation, so that unread* can work correctly. - @always_inline fn __init__(inout self, capacity: Int = io.BUFFER_SIZE): self._capacity = capacity self._size = 0 @@ -59,15 +58,13 @@ struct Buffer( self.offset = 0 self.last_read = OP_INVALID - @always_inline - fn __init__(inout self, owned buf: List[Byte]): + fn __init__(inout self, owned buf: List[UInt8]): self._capacity = buf.capacity self._size = buf.size self._data = buf.steal_data() self.offset = 0 self.last_read = OP_INVALID - @always_inline fn __init__(inout self, owned data: UnsafePointer[UInt8], capacity: Int, size: Int): self._capacity = capacity self._size = size @@ -75,7 +72,6 @@ struct Buffer( self.offset = 0 self.last_read = OP_INVALID - @always_inline fn __moveinit__(inout self, owned other: Self): self._data = other._data self._size = other._size @@ -88,35 +84,38 @@ struct Buffer( other.offset = 0 other.last_read = OP_INVALID - @always_inline fn __del__(owned self): if self._data: self._data.free() - @always_inline fn __len__(self) -> Int: """Returns the number of bytes of the unread portion of the buffer. self._size - self.offset.""" return self._size - self.offset - @always_inline fn bytes_ptr(self) -> UnsafePointer[UInt8]: """Returns a pointer holding the unread portion of the buffer.""" return self._data.offset(self.offset) - @always_inline fn bytes(self) -> List[UInt8]: """Returns a list of bytes holding a copy of the unread portion of the buffer.""" var copy = UnsafePointer[UInt8]().alloc(self._size) memcpy(copy, self._data.offset(self.offset), self._size) return List[UInt8](unsafe_pointer=copy, size=self._size - self.offset, capacity=self._size - self.offset) - @always_inline - fn as_bytes_slice(self: Reference[Self]) -> Span[UInt8, self.is_mutable, self.lifetime]: + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: """Returns the internal data as a Span[UInt8].""" - return Span[UInt8, self.is_mutable, self.lifetime](unsafe_ptr=self[]._data, len=self[]._size) + return Span[UInt8, __lifetime_of(self)](unsafe_ptr=self._data, len=self._size) + + fn as_string_slice(self) -> StringSlice[__lifetime_of(self)]: + """ + Return a StringSlice view of the data owned by the builder. + + Returns: + The string representation of the string builder. Returns an empty string if the string builder is empty. + """ + return StringSlice[__lifetime_of(self)](unsafe_from_utf8_ptr=self._data, len=self._size) - @always_inline fn _resize(inout self, capacity: Int) -> None: """ Resizes the string builder buffer. @@ -132,7 +131,6 @@ struct Buffer( return None - @always_inline fn _resize_if_needed(inout self, bytes_to_add: Int): # TODO: Handle the case where new_capacity is greater than MAX_INT. It should panic. if bytes_to_add > self._capacity - self._size: @@ -141,7 +139,6 @@ struct Buffer( new_capacity = self._capacity + bytes_to_add self._resize(new_capacity) - @always_inline fn __str__(self) -> String: """ Converts the string builder to a string. @@ -150,25 +147,19 @@ struct Buffer( The string representation of the string builder. Returns an empty string if the string builder is empty. """ - var copy = UnsafePointer[UInt8]().alloc(self._size) - memcpy(copy, self._data, self._size) - return StringRef(copy, self._size) + return self.as_string_slice() - @always_inline - fn render(self: Reference[Self]) -> StringSlice[self.is_mutable, self.lifetime]: + @deprecated("Buffer.render() has been deprecated. Use Buffer.as_string_slice() instead.") + fn render(self) -> StringSlice[__lifetime_of(self)]: """ Return a StringSlice view of the data owned by the builder. - Slightly faster than __str__, 10-20% faster in limited testing. Returns: The string representation of the string builder. Returns an empty string if the string builder is empty. """ - return StringSlice[self.is_mutable, self.lifetime]( - unsafe_from_utf8_strref=StringRef(self[]._data, self[]._size) - ) + return self.as_string_slice() - @always_inline - fn _write(inout self, src: Span[Byte]) -> (Int, Error): + fn _write(inout self, src: Span[UInt8]) -> (Int, Error): """ Appends a byte Span to the builder buffer. @@ -182,8 +173,7 @@ struct Buffer( return len(src), Error() - @always_inline - fn write(inout self, src: List[Byte]) -> (Int, Error): + fn write(inout self, src: List[UInt8]) -> (Int, Error): """ Appends a byte List to the builder buffer. @@ -198,7 +188,6 @@ struct Buffer( return bytes_read, err - @always_inline fn write_string(inout self, src: String) -> (Int, Error): """ Appends a string to the builder buffer. @@ -206,10 +195,9 @@ struct Buffer( Args: src: The string to append. """ - return self.write(src.as_bytes_slice()) + return self._write(src.as_bytes_slice()) - @always_inline - fn write_byte(inout self, byte: Byte) -> (Int, Error): + fn write_byte(inout self, byte: UInt8) -> (Int, Error): """Appends the byte c to the buffer, growing the buffer as needed. The returned error is always nil, but is included to match [bufio.Writer]'s write_byte. If the buffer becomes too large, write_byte will panic with @@ -228,12 +216,10 @@ struct Buffer( return 1, Error() - @always_inline fn empty(self) -> Bool: """Reports whether the unread portion of the buffer is empty.""" return self._size <= self.offset - @always_inline fn reset(inout self): """Resets the buffer to be empty, but it retains the underlying storage for use by future writes. @@ -245,8 +231,7 @@ struct Buffer( self.offset = 0 self.last_read = OP_INVALID - @always_inline - fn _read(inout self, inout dest: Span[Byte, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Reads the next len(dest) bytes from the buffer or until the buffer is drained. The return value n is the number of bytes read. If the buffer has no data to return, err is io.EOF (unless len(dest) is zero); @@ -269,7 +254,8 @@ struct Buffer( return 0, io.EOF # Copy the data of the internal buffer from offset to len(buf) into the destination buffer at the given index. - var bytes_read = copy(dest, self.as_bytes_slice()[self.offset :]) + var bytes_to_read = self.as_bytes_slice()[self.offset :] + var bytes_read = copy(dest.unsafe_ptr(), bytes_to_read.unsafe_ptr(), source_length=len(bytes_to_read)) dest._len += bytes_read self.offset += bytes_read @@ -278,8 +264,7 @@ struct Buffer( return bytes_read, Error() - @always_inline - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): """Reads the next len(dest) bytes from the buffer or until the buffer is drained. The return value n is the number of bytes read. If the buffer has no data to return, err is io.EOF (unless len(dest) is zero); @@ -300,15 +285,14 @@ struct Buffer( return bytes_read, err - @always_inline - fn read_byte(inout self) -> (Byte, Error): + fn read_byte(inout self) -> (UInt8, Error): """Reads and returns the next byte from the buffer. If no byte is available, it returns error io.EOF. """ if self.empty(): # Buffer is empty, reset to recover space. self.reset() - return Byte(0), io.EOF + return UInt8(0), io.EOF var byte = self._data[self.offset] self.offset += 1 @@ -316,7 +300,6 @@ struct Buffer( return byte, Error() - @always_inline fn unread_byte(inout self) -> Error: """Unreads the last byte returned by the most recent successful read operation that read at least one byte. If a write has happened since @@ -332,8 +315,7 @@ struct Buffer( return Error() - @always_inline - fn read_bytes(inout self, delim: Byte) -> (List[Byte], Error): + fn read_bytes(inout self, delim: UInt8) -> (List[UInt8], Error): """Reads until the first occurrence of delim in the input, returning a slice containing the data up to and including the delimiter. If read_bytes encounters an error before finding a delimiter, @@ -345,44 +327,42 @@ struct Buffer( delim: The delimiter to read until. Returns: - A List[Byte] struct containing the data up to and including the delimiter. + A List[UInt8] struct containing the data up to and including the delimiter. """ - var slice: Span[UInt8, True, __lifetime_of(self)] + var slice: Span[UInt8, __lifetime_of(self)] var err: Error slice, err = self.read_slice(delim) - var bytes = List[Byte](capacity=len(slice) + 1) + var bytes = List[UInt8](capacity=len(slice) + 1) for byte in slice: bytes.append(byte[]) return bytes, err - @always_inline - fn read_slice(self: Reference[Self, True], delim: Byte) -> (Span[UInt8, self.is_mutable, self.lifetime], Error): + fn read_slice(inout self, delim: UInt8) -> (Span[UInt8, __lifetime_of(self)], Error): """Like read_bytes but returns a reference to internal buffer data. Args: delim: The delimiter to read until. Returns: - A List[Byte] struct containing the data up to and including the delimiter. + A List[UInt8] struct containing the data up to and including the delimiter. """ - var i = index_byte(bytes=self[].as_bytes_slice(), delim=delim) - var end = self[].offset + i + 1 + var i = index_byte(bytes=self.as_bytes_slice(), delim=delim) + var end = self.offset + i + 1 var err = Error() if i < 0: - end = self[]._size + end = self._size err = Error(str(io.EOF)) - var line = self[].as_bytes_slice()[self[].offset : end] - self[].offset = end - self[].last_read = OP_READ + var line = self.as_bytes_slice()[self.offset : end] + self.offset = end + self.last_read = OP_READ return line, err - @always_inline - fn read_string(inout self, delim: Byte) -> (String, Error): + fn read_string(inout self, delim: UInt8) -> (String, Error): """Reads until the first occurrence of delim in the input, returning a string containing the data up to and including the delimiter. If read_string encounters an error before finding a delimiter, @@ -403,8 +383,7 @@ struct Buffer( return String(bytes), err - @always_inline - fn next(self: Reference[Self, True], number_of_bytes: Int) raises -> Span[Byte, self.is_mutable, self.lifetime]: + fn next(inout self, number_of_bytes: Int) -> Span[UInt8, __lifetime_of(self)]: """Returns a slice containing the next n bytes from the buffer, advancing the buffer as if the bytes had been returned by [Buffer.read]. If there are fewer than n bytes in the buffer, next returns the entire buffer. @@ -416,17 +395,17 @@ struct Buffer( Returns: A slice containing the next n bytes from the buffer. """ - self[].last_read = OP_INVALID - var bytes_remaining = len(self[]) + self.last_read = OP_INVALID + var bytes_remaining = len(self) var bytes_to_read = number_of_bytes if bytes_to_read > bytes_remaining: bytes_to_read = bytes_remaining - var data = self[].as_bytes_slice()[self[].offset : self[].offset + bytes_to_read] + var data = self.as_bytes_slice()[self.offset : self.offset + bytes_to_read] - self[].offset += bytes_to_read + self.offset += bytes_to_read if bytes_to_read > 0: - self[].last_read = OP_READ + self.last_read = OP_READ return data @@ -477,11 +456,11 @@ fn new_buffer(capacity: Int = io.BUFFER_SIZE) -> Buffer: In most cases, new([Buffer]) (or just declaring a [Buffer] variable) is sufficient to initialize a [Buffer]. """ - var b = List[Byte](capacity=capacity) + var b = List[UInt8](capacity=capacity) return Buffer(b^) -fn new_buffer(owned buf: List[Byte]) -> Buffer: +fn new_buffer(owned buf: List[UInt8]) -> Buffer: """Creates and initializes a new [Buffer] using buf as its` initial contents. The new [Buffer] takes ownership of buf, and the caller should not use buf after this call. new_buffer is intended to diff --git a/gojo/bytes/reader.mojo b/gojo/bytes/reader.mojo index 8948b9a..f640ec3 100644 --- a/gojo/bytes/reader.mojo +++ b/gojo/bytes/reader.mojo @@ -26,7 +26,6 @@ struct Reader( var index: Int # current reading index var prev_rune: Int # index of previous rune; or < 0 - @always_inline fn __init__(inout self, owned buffer: List[UInt8]): """Initializes a new [Reader.Reader] struct.""" self.capacity = buffer.capacity @@ -35,7 +34,6 @@ struct Reader( self.index = 0 self.prev_rune = -1 - @always_inline fn __moveinit__(inout self, owned other: Reader): """Initializes a new [Reader.Reader] struct by moving the data from another [Reader.Reader] struct.""" self.capacity = other.capacity @@ -50,24 +48,19 @@ struct Reader( other.index = 0 other.prev_rune = -1 - @always_inline fn __len__(self) -> Int: - """len returns the number of bytes of the unread portion of the - slice.""" + """Returns the number of bytes of the unread portion of the slice.""" return self.size - int(self.index) - @always_inline fn __del__(owned self): if self.data: self.data.free() - @always_inline - fn as_bytes_slice(self: Reference[Self]) -> Span[UInt8, self.is_mutable, self.lifetime]: + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: """Returns the internal data as a Span[UInt8].""" - return Span[UInt8, self.is_mutable, self.lifetime](unsafe_ptr=self[].data, len=self[].size) + return Span[UInt8, __lifetime_of(self)](unsafe_ptr=self.data, len=self.size) - @always_inline - fn _read(inout self, inout dest: Span[UInt8, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Reads from the internal buffer into the dest List[UInt8] struct. Implements the [io.Reader] Interface. @@ -83,13 +76,13 @@ struct Reader( # Copy the data of the internal buffer from offset to len(buf) into the destination buffer at the given index. self.prev_rune = -1 - var bytes_written = copy(dest, self.as_bytes_slice()[self.index : self.size], len(dest)) + var bytes_to_write = self.as_bytes_slice()[self.index : self.size] + var bytes_written = copy(dest.unsafe_ptr(), bytes_to_write.unsafe_ptr(), len(bytes_to_write)) dest._len += bytes_written self.index += bytes_written return bytes_written, Error() - @always_inline fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): """Reads from the internal buffer into the dest List[UInt8] struct. Implements the [io.Reader] Interface. @@ -108,14 +101,14 @@ struct Reader( return bytes_read, err - @always_inline - fn _read_at(self, inout dest: Span[UInt8, True], off: Int, capacity: Int) -> (Int, Error): + fn _read_at(self, inout dest: Span[UInt8], off: Int, capacity: Int) -> (Int, Error): """Reads len(dest) bytes into dest beginning at byte offset off. Implements the [io.ReaderAt] Interface. Args: - dest: The destination List[UInt8] struct to read into. + dest: The destination Span[UInt8] struct to read into. off: The offset to start reading from. + capacity: The capacity of the destination buffer. Returns: Int: The number of bytes read into dest. @@ -128,13 +121,12 @@ struct Reader( return 0, io.EOF var unread_bytes = self.as_bytes_slice()[off : self.size] - var bytes_written = copy(dest, unread_bytes) + var bytes_written = copy(dest.unsafe_ptr(), unread_bytes.unsafe_ptr(), len(unread_bytes)) if bytes_written < len(dest): return 0, io.EOF return bytes_written, Error() - @always_inline fn read_at(self, inout dest: List[UInt8], off: Int) -> (Int, Error): """Reads len(dest) bytes into dest beginning at byte offset off. Implements the [io.ReaderAt] Interface. @@ -155,7 +147,6 @@ struct Reader( return bytes_read, err - @always_inline fn read_byte(inout self) -> (UInt8, Error): """Reads and returns a single byte from the internal buffer. Implements the [io.ByteReader] Interface.""" self.prev_rune = -1 @@ -166,7 +157,6 @@ struct Reader( self.index += 1 return byte, Error() - @always_inline fn unread_byte(inout self) -> Error: """Unreads the last byte read by moving the read position back by one. Complements [Reader.read_byte] in implementing the [io.ByteScanner] Interface. @@ -206,7 +196,6 @@ struct Reader( # self.prev_rune = -1 # return nil - @always_inline fn seek(inout self, offset: Int, whence: Int) -> (Int, Error): """Moves the read position to the specified offset from the specified whence. @@ -235,7 +224,6 @@ struct Reader( self.index = position return position, Error() - @always_inline fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int, Error): """Writes data to w until the buffer is drained or an error occurs. implements the [io.WriterTo] Interface. @@ -260,7 +248,6 @@ struct Reader( return write_count, Error() - @always_inline fn reset(inout self, owned buffer: List[UInt8]): """Resets the [Reader.Reader] to be reading from buffer. diff --git a/gojo/fmt/fmt.mojo b/gojo/fmt/fmt.mojo index 6766715..5413c61 100644 --- a/gojo/fmt/fmt.mojo +++ b/gojo/fmt/fmt.mojo @@ -9,8 +9,6 @@ Boolean Integer %d base 10 %q a single-quoted character literal. -%x base 16, with lower-case letters for a-f -%X base 16, with upper-case letters for A-F Floating-point and complex constituents: %f decimal point but no exponent, e.g. 123.456 @@ -28,9 +26,8 @@ TODO: from utils.variant import Variant from math import floor -from ..builtins import Byte -alias Args = Variant[String, Int, Float64, Bool, List[Byte]] +alias Args = Variant[String, Int, Float64, Bool, List[UInt8]] fn replace_first(s: String, old: String, new: String) -> String: @@ -77,51 +74,6 @@ fn find_first_verb(s: String, verbs: List[String]) -> String: return verb -alias BASE10_TO_BASE16 = List[String]( - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "a", - "b", - "c", - "d", - "e", - "f", -) - - -fn convert_base10_to_base16(value: Int) -> String: - """Converts a base 10 number to base 16. - - Args: - value: Base 10 number. - - Returns: - Base 16 number as a String. - """ - - var val: Float64 = 0.0 - var result: Float64 = value - var base16: String = "" - while result > 1: - var temp = result / 16 - var floor_result = floor(temp) - var remainder = temp - floor_result - result = floor_result - val = 16 * remainder - - base16 = BASE10_TO_BASE16[int(val)] + base16 - - return base16 - - fn format_string(format: String, arg: String) -> String: var verb = find_first_verb(format, List[String]("%s", "%q")) var arg_to_place = arg @@ -131,7 +83,7 @@ fn format_string(format: String, arg: String) -> String: return replace_first(format, String("%s"), arg) -fn format_bytes(format: String, arg: List[Byte]) -> String: +fn format_bytes(format: String, arg: List[UInt8]) -> String: var argument = arg if argument[-1] != 0: argument.append(0) @@ -140,13 +92,9 @@ fn format_bytes(format: String, arg: List[Byte]) -> String: fn format_integer(format: String, arg: Int) -> String: - var verb = find_first_verb(format, List[String]("%x", "%X", "%d", "%q")) + var verb = find_first_verb(format, List[String]("%d", "%q")) var arg_to_place = str(arg) - if verb == "%x": - arg_to_place = str(convert_base10_to_base16(arg)).lower() - elif verb == "%X": - arg_to_place = str(convert_base10_to_base16(arg)).upper() - elif verb == "%q": + if verb == "%q": arg_to_place = "'" + str(arg) + "'" return replace_first(format, verb, arg_to_place) @@ -180,8 +128,8 @@ fn sprintf(formatting: String, *args: Args) -> String: var argument = args[i] if argument.isa[String](): text = format_string(text, argument[String]) - elif argument.isa[List[Byte]](): - text = format_bytes(text, argument[List[Byte]]) + elif argument.isa[List[UInt8]](): + text = format_bytes(text, argument[List[UInt8]]) elif argument.isa[Int](): text = format_integer(text, argument[Int]) elif argument.isa[Float64](): @@ -222,8 +170,8 @@ fn printf(formatting: String, *args: Args) raises: var argument = args[i] if argument.isa[String](): text = format_string(text, argument[String]) - elif argument.isa[List[Byte]](): - text = format_bytes(text, argument[List[Byte]]) + elif argument.isa[List[UInt8]](): + text = format_bytes(text, argument[List[UInt8]]) elif argument.isa[Int](): text = format_integer(text, argument[Int]) elif argument.isa[Float64](): diff --git a/gojo/io/__init__.mojo b/gojo/io/__init__.mojo index fb4e9b6..18c5f9b 100644 --- a/gojo/io/__init__.mojo +++ b/gojo/io/__init__.mojo @@ -83,7 +83,7 @@ trait Reader(Movable): fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): ... - fn _read(inout self, inout dest: Span[UInt8, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): ... @@ -99,8 +99,8 @@ trait Writer(Movable): Implementations must not retain p. """ - # fn _write(inout self, src: Span[UInt8]) -> (Int, Error): - # ... + fn _write(inout self, src: Span[UInt8]) -> (Int, Error): + ... fn write(inout self, src: List[UInt8]) -> (Int, Error): ... @@ -238,7 +238,7 @@ trait ReaderAt: fn read_at(self, inout dest: List[UInt8], off: Int) -> (Int, Error): ... - fn _read_at(self, inout dest: Span[UInt8, True], off: Int, capacity: Int) -> (Int, Error): + fn _read_at(self, inout dest: Span[UInt8], off: Int, capacity: Int) -> (Int, Error): ... diff --git a/gojo/io/file.mojo b/gojo/io/file.mojo index 443ae93..2c53dad 100644 --- a/gojo/io/file.mojo +++ b/gojo/io/file.mojo @@ -1,27 +1,22 @@ import ..io from ..builtins import copy -from ..syscall import FileDescriptorBase -struct FileWrapper(FileDescriptorBase, io.ByteReader): +struct FileWrapper(io.ReadWriteCloser, io.ByteReader): var handle: FileHandle - @always_inline fn __init__(inout self, path: String, mode: String) raises: self.handle = open(path, mode) - @always_inline fn __moveinit__(inout self, owned existing: Self): self.handle = existing.handle^ - @always_inline fn __del__(owned self): var err = self.close() if err: # TODO: __del__ can't raise, but there should be some fallback. print(str(err)) - @always_inline fn close(inout self) -> Error: try: self.handle.close() @@ -30,8 +25,7 @@ struct FileWrapper(FileDescriptorBase, io.ByteReader): return Error() - @always_inline - fn _read(inout self, inout dest: Span[UInt8, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8, _], capacity: Int) -> (Int, Error): """Read from the file handle into dest's pointer. Pretty hacky way to force the filehandle read into the defined trait, and it's unsafe since we're reading directly into the pointer. @@ -47,14 +41,18 @@ struct FileWrapper(FileDescriptorBase, io.ByteReader): except e: return 0, e - _ = copy(dest, Span(result), len(dest)) + var count = 0 + var target = dest.unsafe_ptr() + len(dest) + for i in range(len(result)): + target[i] = result[i] + count += 1 + dest._len += count if bytes_read == 0: return bytes_read, io.EOF return bytes_read, Error() - @always_inline fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): """Read from the file handle into dest's pointer. Pretty hacky way to force the filehandle read into the defined trait, and it's unsafe since we're @@ -78,7 +76,6 @@ struct FileWrapper(FileDescriptorBase, io.ByteReader): return bytes_read, Error() - @always_inline fn read_all(inout self) -> (List[UInt8], Error): var bytes = List[UInt8](capacity=io.BUFFER_SIZE) while True: @@ -93,7 +90,6 @@ struct FileWrapper(FileDescriptorBase, io.ByteReader): if len(temp) < io.BUFFER_SIZE: return bytes, io.EOF - @always_inline fn read_byte(inout self) -> (UInt8, Error): try: var bytes: List[UInt8] @@ -103,14 +99,12 @@ struct FileWrapper(FileDescriptorBase, io.ByteReader): except e: return UInt8(0), e - @always_inline fn read_bytes(inout self, size: Int = -1) raises -> (List[UInt8], Error): try: return self.handle.read_bytes(size), Error() except e: return List[UInt8](), e - @always_inline fn stream_until_delimiter(inout self, inout dest: List[UInt8], delimiter: UInt8, max_size: Int) -> Error: var byte: UInt8 var err = Error() @@ -124,7 +118,6 @@ struct FileWrapper(FileDescriptorBase, io.ByteReader): dest.append(byte) return Error("Stream too long") - @always_inline fn seek(inout self, offset: Int, whence: Int = 0) -> (Int, Error): try: var position = self.handle.seek(UInt64(offset), whence) @@ -132,7 +125,6 @@ struct FileWrapper(FileDescriptorBase, io.ByteReader): except e: return 0, e - @always_inline fn _write(inout self, src: Span[UInt8]) -> (Int, Error): if len(src) == 0: return 0, Error("No data to write") @@ -143,6 +135,5 @@ struct FileWrapper(FileDescriptorBase, io.ByteReader): except e: return 0, Error(str(e)) - @always_inline fn write(inout self, src: List[UInt8]) -> (Int, Error): return self._write(Span(src)) diff --git a/gojo/io/io.mojo b/gojo/io/io.mojo index 87b7da2..5c9dd9e 100644 --- a/gojo/io/io.mojo +++ b/gojo/io/io.mojo @@ -1,4 +1,4 @@ -from ..builtins import copy, Byte, panic +from ..builtins import copy, panic alias BUFFER_SIZE = 4096 @@ -32,7 +32,7 @@ fn write_string[W: StringWriter](inout writer: W, string: String) -> (Int, Error return writer.write_string(string) -fn read_at_least[R: Reader](inout reader: R, inout dest: List[Byte], min: Int) -> (Int, Error): +fn read_at_least[R: Reader](inout reader: R, inout dest: List[UInt8], min: Int) -> (Int, Error): """Reads from r into buf until it has read at least min bytes. It returns the number of bytes copied and an error if fewer bytes were read. The error is EOF only if no bytes were read. @@ -68,7 +68,7 @@ fn read_at_least[R: Reader](inout reader: R, inout dest: List[Byte], min: Int) - return total_bytes_read, error -fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error): +fn read_full[R: Reader](inout reader: R, inout dest: List[UInt8]) -> (Int, Error): """Reads exactly len(buf) bytes from r into buf. It returns the number of bytes copied and an error if fewer bytes were read. The error is EOF only if no bytes were read. @@ -130,7 +130,7 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # } -# fn copy_buffer[W: Writer, R: Reader](dst: W, src: R, buf: Span[Byte]) raises -> Int: +# fn copy_buffer[W: Writer, R: Reader](dst: W, src: R, buf: Span[UInt8]) raises -> Int: # """Actual implementation of copy and CopyBuffer. # if buf is nil, one is allocated. # """ @@ -150,11 +150,11 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # return written -# fn copy_buffer[W: Writer, R: ReaderWriteTo](dst: W, src: R, buf: Span[Byte]) -> Int: +# fn copy_buffer[W: Writer, R: ReaderWriteTo](dst: W, src: R, buf: Span[UInt8]) -> Int: # return src.write_to(dst) -# fn copy_buffer[W: WriterReadFrom, R: Reader](dst: W, src: R, buf: Span[Byte]) -> Int: +# fn copy_buffer[W: WriterReadFrom, R: Reader](dst: W, src: R, buf: Span[UInt8]) -> Int: # return dst.read_from(src) # # LimitReader returns a Reader that reads from r @@ -405,7 +405,7 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # TODO: read directly into dest -fn read_all[R: Reader](inout reader: R) -> (List[Byte], Error): +fn read_all[R: Reader](inout reader: R) -> (List[UInt8], Error): """Reads from r until an error or EOF and returns the data it read. A successful call returns err == nil, not err == EOF. Because ReadAll is defined to read from src until EOF, it does not treat an EOF from Read @@ -416,11 +416,11 @@ fn read_all[R: Reader](inout reader: R) -> (List[Byte], Error): Returns: The data read.""" - var dest = List[Byte](capacity=BUFFER_SIZE) + var dest = List[UInt8](capacity=BUFFER_SIZE) var at_eof: Bool = False while True: - var temp = List[Byte](capacity=BUFFER_SIZE) + var temp = List[UInt8](capacity=BUFFER_SIZE) var bytes_read: Int var err: Error bytes_read, err = reader.read(temp) diff --git a/gojo/io/std.mojo b/gojo/io/std.mojo index 553be78..3a11739 100644 --- a/gojo/io/std.mojo +++ b/gojo/io/std.mojo @@ -1,5 +1,4 @@ import ..io -from ..syscall import FD @value @@ -9,7 +8,7 @@ struct STDWriter[file_descriptor: Int](Copyable, io.Writer, io.StringWriter): @always_inline fn __init__(inout self): constrained[ - file_descriptor == FD.STDOUT or file_descriptor == FD.STDERR, + file_descriptor == 1 or file_descriptor == 2, "The STDWriter Struct is meant to write to STDOUT and STDERR. file_descriptor must be 1 or 2.", ]() diff --git a/gojo/net/fd.mojo b/gojo/net/fd.mojo index 5511422..e5fb8b8 100644 --- a/gojo/net/fd.mojo +++ b/gojo/net/fd.mojo @@ -13,24 +13,20 @@ struct FileDescriptor(FileDescriptorBase): var fd: Int var is_closed: Bool - @always_inline fn __init__(inout self, fd: Int): self.fd = fd self.is_closed = False - @always_inline fn __moveinit__(inout self, owned existing: Self): self.fd = existing.fd self.is_closed = existing.is_closed - @always_inline fn __del__(owned self): if not self.is_closed: var err = self.close() if err: print(str(err)) - @always_inline fn close(inout self) -> Error: """Mark the file descriptor as closed.""" var close_status = close(self.fd) @@ -40,8 +36,7 @@ struct FileDescriptor(FileDescriptorBase): self.is_closed = True return Error() - @always_inline - fn _read(inout self, inout dest: Span[UInt8, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Receive data from the file descriptor and write it to the buffer provided.""" var bytes_received = recv( self.fd, @@ -58,7 +53,6 @@ struct FileDescriptor(FileDescriptorBase): return bytes_received, Error() - @always_inline fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): """Receive data from the file descriptor and write it to the buffer provided.""" var span = Span(dest) @@ -70,12 +64,10 @@ struct FileDescriptor(FileDescriptorBase): return bytes_read, err - @always_inline fn write(inout self, src: List[UInt8]) -> (Int, Error): """Write data from the buffer to the file descriptor.""" return self._write(Span(src)) - @always_inline fn _write(inout self, src: Span[UInt8]) -> (Int, Error): """Write data from the buffer to the file descriptor.""" var bytes_sent = send(self.fd, src.unsafe_ptr(), len(src), 0) diff --git a/gojo/net/ip.mojo b/gojo/net/ip.mojo index 4af7748..7fe8dbe 100644 --- a/gojo/net/ip.mojo +++ b/gojo/net/ip.mojo @@ -39,7 +39,7 @@ fn get_addr_info(host: String) raises -> AddrInfo: ) var status = getaddrinfo( - host.unsafe_uint8_ptr(), + host.unsafe_ptr(), UnsafePointer[UInt8](), UnsafePointer.address_of(hints), UnsafePointer.address_of(servinfo), @@ -51,7 +51,7 @@ fn get_addr_info(host: String) raises -> AddrInfo: print("servinfo is null") raise Error("Failed to get address info. Pointer to addrinfo is null.") - return move_from_pointee(servinfo) + return servinfo.take_pointee() elif os_is_linux(): var servinfo = UnsafePointer[addrinfo_unix]().alloc(1) servinfo[0] = addrinfo_unix() @@ -62,7 +62,7 @@ fn get_addr_info(host: String) raises -> AddrInfo: ) var status = getaddrinfo_unix( - host.unsafe_uint8_ptr(), + host.unsafe_ptr(), UnsafePointer[UInt8](), UnsafePointer.address_of(hints), UnsafePointer.address_of(servinfo), @@ -74,7 +74,7 @@ fn get_addr_info(host: String) raises -> AddrInfo: print("servinfo is null") raise Error("Failed to get address info. Pointer to addrinfo is null.") - return move_from_pointee(servinfo) + return servinfo.take_pointee() else: raise Error("Windows is not supported yet! Sorry!") @@ -102,7 +102,7 @@ fn get_ip_address(host: String) raises -> String: raise Error("Failed to get IP address. getaddrinfo was called successfully, but ai_addr is null.") # Cast sockaddr struct to sockaddr_in struct and convert the binary IP to a string using inet_ntop. - var addr_in = move_from_pointee(ai_addr.bitcast[sockaddr_in]()) + var addr_in = ai_addr.bitcast[sockaddr_in]().take_pointee() return convert_binary_ip_to_string(addr_in.sin_addr.s_addr, address_family, address_length).strip() @@ -117,11 +117,11 @@ fn convert_binary_port_to_int(port: UInt16) -> Int: fn convert_ip_to_binary(ip_address: String, address_family: Int) -> UInt32: var ip_buffer = UnsafePointer[UInt8].alloc(4) - var status = inet_pton(address_family, ip_address.unsafe_uint8_ptr(), ip_buffer) + var status = inet_pton(address_family, ip_address.unsafe_ptr(), ip_buffer) if status == -1: print("Failed to convert IP address to binary") - return move_from_pointee(ip_buffer.bitcast[c_uint]()) + return ip_buffer.bitcast[c_uint]().take_pointee() fn convert_binary_ip_to_string(owned ip_address: UInt32, address_family: Int32, address_length: UInt32) -> String: @@ -175,7 +175,7 @@ fn convert_sockaddr_to_host_port(sockaddr: UnsafePointer[sockaddr]) -> (HostPort return HostPort(), Error("sockaddr is null, nothing to convert.") # Cast sockaddr struct to sockaddr_in to convert binary IP to string. - var addr_in = move_from_pointee(sockaddr.bitcast[sockaddr_in]()) + var addr_in = sockaddr.bitcast[sockaddr_in]().take_pointee() return ( HostPort( diff --git a/gojo/net/socket.mojo b/gojo/net/socket.mojo index f21867d..d031792 100644 --- a/gojo/net/socket.mojo +++ b/gojo/net/socket.mojo @@ -159,23 +159,18 @@ struct Socket(FileDescriptorBase): if err: print("Failed to close socket during deletion:", str(err)) - @always_inline fn local_address_as_udp(self) -> UDPAddr: return UDPAddr(self.local_address) - @always_inline fn local_address_as_tcp(self) -> TCPAddr: return TCPAddr(self.local_address) - @always_inline fn remote_address_as_udp(self) -> UDPAddr: return UDPAddr(self.remote_address) - @always_inline fn remote_address_as_tcp(self) -> TCPAddr: return TCPAddr(self.remote_address) - @always_inline fn accept(self) raises -> Socket: """Accept a connection. The socket must be bound to an address and listening for connections. The return value is a connection where conn is a new socket object usable to send and receive data on the connection, @@ -218,7 +213,6 @@ struct Socket(FileDescriptorBase): if listen(self.fd.fd, queued) == -1: raise Error("Failed to listen for connections") - @always_inline fn bind(inout self, address: String, port: Int) raises: """Bind the socket to address. The socket must not already be bound. (The format of address depends on the address family). @@ -243,12 +237,10 @@ struct Socket(FileDescriptorBase): var local = self.get_sock_name() self.local_address = BaseAddr(local.host, local.port) - @always_inline fn file_no(self) -> Int32: """Return the file descriptor of the socket.""" return self.fd.fd - @always_inline fn get_sock_name(self) raises -> HostPort: """Return the address of the socket.""" if self._closed: @@ -265,7 +257,7 @@ struct Socket(FileDescriptorBase): ) if status == -1: raise Error("Socket.get_sock_name: Failed to get address of local socket.") - var addr_in = move_from_pointee(local_address_ptr.bitcast[sockaddr_in]()) + var addr_in = local_address_ptr.bitcast[sockaddr_in]().take_pointee() return HostPort( host=convert_binary_ip_to_string(addr_in.sin_addr.s_addr, AddressFamily.AF_INET, 16), @@ -315,7 +307,7 @@ struct Socket(FileDescriptorBase): if status == -1: raise Error("Socket.get_sock_opt failed with status: " + str(status)) - return move_from_pointee(option_value_pointer.bitcast[Int]()) + return option_value_pointer.bitcast[Int]().take_pointee() fn set_socket_option(self, option_name: Int, owned option_value: UInt8 = 1) raises: """Return the value of the given socket option. @@ -358,7 +350,6 @@ struct Socket(FileDescriptorBase): self.remote_address = BaseAddr(remote.host, remote.port) return Error() - @always_inline fn _write(inout self: Self, src: Span[UInt8]) -> (Int, Error): """Send data to the socket. The socket must be connected to a remote socket. @@ -370,7 +361,6 @@ struct Socket(FileDescriptorBase): """ return self.fd._write(src) - @always_inline fn write(inout self: Self, src: List[UInt8]) -> (Int, Error): """Send data to the socket. The socket must be connected to a remote socket. @@ -411,7 +401,6 @@ struct Socket(FileDescriptorBase): return Error() - @always_inline fn send_to(inout self, src: Span[UInt8], address: String, port: Int) -> (Int, Error): """Send data to the a remote address by connecting to the remote socket before sending. The socket must be not already be connected to a remote socket. @@ -435,7 +424,6 @@ struct Socket(FileDescriptorBase): return bytes_sent, Error() - @always_inline fn receive(inout self, size: Int = io.BUFFER_SIZE) -> (List[UInt8], Error): """Receive data from the socket into the buffer with capacity of `size` bytes. @@ -461,8 +449,7 @@ struct Socket(FileDescriptorBase): return bytes, Error() - @always_inline - fn _read(inout self, inout dest: Span[UInt8, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Receive data from the socket into the buffer dest. Equivalent to recv_into(). Args: @@ -474,7 +461,6 @@ struct Socket(FileDescriptorBase): """ return self.fd._read(dest, capacity) - @always_inline fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): """Receive data from the socket into the buffer dest. Equivalent to recv_into(). @@ -493,7 +479,6 @@ struct Socket(FileDescriptorBase): return bytes_read, err - @always_inline fn receive_from(inout self, size: Int = io.BUFFER_SIZE) -> (List[UInt8], HostPort, Error): """Receive data from the socket into the buffer dest. @@ -530,7 +515,6 @@ struct Socket(FileDescriptorBase): return bytes, remote, Error() - @always_inline fn receive_from_into(inout self, inout dest: List[UInt8]) -> (Int, HostPort, Error): """Receive data from the socket into the buffer dest.""" var remote_address_ptr = UnsafePointer[sockaddr].alloc(1) @@ -559,11 +543,9 @@ struct Socket(FileDescriptorBase): return bytes_read, remote, Error() - @always_inline fn shutdown(self): _ = shutdown(self.fd.fd, SHUT_RDWR) - @always_inline fn close(inout self) -> Error: """Mark the socket closed. Once that happens, all future operations on the socket object will fail. @@ -590,7 +572,6 @@ struct Socket(FileDescriptorBase): # """ # self.set_socket_option(SocketOptions.SO_RCVTIMEO, duration) - @always_inline fn send_file(self, file: FileHandle) -> Error: try: var bytes = file.read_bytes() diff --git a/gojo/net/tcp.mojo b/gojo/net/tcp.mojo index 148565e..e56cb72 100644 --- a/gojo/net/tcp.mojo +++ b/gojo/net/tcp.mojo @@ -55,7 +55,7 @@ struct TCPConnection(Movable): self.socket = existing.socket^ @always_inline - fn _read(inout self, inout dest: Span[UInt8, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Reads data from the underlying file descriptor. Args: diff --git a/gojo/net/udp.mojo b/gojo/net/udp.mojo index ebc0fc0..ca2e710 100644 --- a/gojo/net/udp.mojo +++ b/gojo/net/udp.mojo @@ -63,7 +63,7 @@ struct UDPConnection(Movable): var err = Error() bytes_read, remote, err = self.socket.receive_from_into(dest) if err: - if str(err) != io.EOF: + if str(err) != str(io.EOF): return bytes_read, remote, err return bytes_read, remote, err diff --git a/gojo/strings/builder.mojo b/gojo/strings/builder.mojo index 24ac6cd..39f3004 100644 --- a/gojo/strings/builder.mojo +++ b/gojo/strings/builder.mojo @@ -36,14 +36,12 @@ struct StringBuilder[growth_factor: Float32 = 2]( var _size: Int var _capacity: Int - @always_inline fn __init__(inout self, *, capacity: Int = 4096): constrained[growth_factor >= 1.25]() self._data = UnsafePointer[UInt8]().alloc(capacity) self._size = 0 self._capacity = capacity - @always_inline fn __moveinit__(inout self, owned other: Self): self._data = other._data self._size = other._size @@ -52,17 +50,14 @@ struct StringBuilder[growth_factor: Float32 = 2]( other._size = 0 other._capacity = 0 - @always_inline fn __del__(owned self): if self._data: self._data.free() - @always_inline fn __len__(self) -> Int: """Returns the length of the string builder.""" return self._size - @always_inline fn __str__(self) -> String: """ Converts the string builder to a string. @@ -75,15 +70,11 @@ struct StringBuilder[growth_factor: Float32 = 2]( memcpy(copy, self._data, self._size) return StringRef(copy, self._size) - @always_inline - fn as_bytes_slice(self: Reference[Self]) -> Span[UInt8, self.is_mutable, self.lifetime]: - """Returns the internal _data as a Span[UInt8].""" - return Span[UInt8, self.is_mutable, self.lifetime](unsafe_ptr=self[]._data, len=self[]._size) + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](unsafe_ptr=self._data, len=self._size) - @always_inline - fn render( - self: Reference[Self], - ) -> StringSlice[self.is_mutable, self.lifetime]: + fn render(ref [_]self) -> StringSlice[__lifetime_of(self)]: """ Return a StringSlice view of the _data owned by the builder. Slightly faster than __str__, 10-20% faster in limited testing. @@ -91,9 +82,8 @@ struct StringBuilder[growth_factor: Float32 = 2]( Returns: The string representation of the string builder. Returns an empty string if the string builder is empty. """ - return StringSlice[self.is_mutable, self.lifetime](unsafe_from_utf8_ptr=self[]._data, len=self[]._size) + return StringSlice[__lifetime_of(self)](unsafe_from_utf8_ptr=self._data, len=self._size) - @always_inline fn _resize(inout self, _capacity: Int) -> None: """ Resizes the string builder buffer. @@ -109,7 +99,6 @@ struct StringBuilder[growth_factor: Float32 = 2]( return None - @always_inline fn _resize_if_needed(inout self, bytes_to_add: Int): """Resizes the buffer if the bytes to add exceeds the current capacity.""" # TODO: Handle the case where new_capacity is greater than MAX_INT. It should panic. @@ -119,7 +108,6 @@ struct StringBuilder[growth_factor: Float32 = 2]( new_capacity = self._capacity + bytes_to_add self._resize(new_capacity) - @always_inline fn _write(inout self, src: Span[UInt8]) -> (Int, Error): """ Appends a byte Span to the builder buffer. @@ -133,7 +121,6 @@ struct StringBuilder[growth_factor: Float32 = 2]( return len(src), Error() - @always_inline fn write(inout self, src: List[UInt8]) -> (Int, Error): """ Appends a byte List to the builder buffer. @@ -149,7 +136,6 @@ struct StringBuilder[growth_factor: Float32 = 2]( return bytes_read, err - @always_inline fn write_string(inout self, src: String) -> (Int, Error): """ Appends a string to the builder buffer. @@ -159,7 +145,6 @@ struct StringBuilder[growth_factor: Float32 = 2]( """ return self._write(src.as_bytes_slice()) - @always_inline fn write_byte(inout self, byte: UInt8) -> (Int, Error): self._resize_if_needed(1) self._data[self._size] = byte diff --git a/gojo/strings/reader.mojo b/gojo/strings/reader.mojo index 97ca0d9..a2019ff 100644 --- a/gojo/strings/reader.mojo +++ b/gojo/strings/reader.mojo @@ -21,25 +21,18 @@ struct Reader( var read_pos: Int # current reading index var prev_rune: Int # index of previous rune; or < 0 - @always_inline fn __init__(inout self, string: String = ""): self.string = string self.read_pos = 0 self.prev_rune = -1 - @always_inline fn __len__(self) -> Int: - """Returns the number of bytes of the unread portion of the string. - - Returns: - int: the number of bytes of the unread portion of the string. - """ + """Returns the number of bytes of the unread portion of the string.""" if self.read_pos >= len(self.string): return 0 return len(self.string) - self.read_pos - @always_inline fn size(self) -> Int: """Returns the original length of the underlying string. size is the number of bytes available for reading via [Reader.read_at]. @@ -51,8 +44,7 @@ struct Reader( """ return len(self.string) - @always_inline - fn _read(inout self, inout dest: Span[UInt8, True], capacity: Int) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Reads from the underlying string into the provided List[UInt8] object. Implements the [io.Reader] trait. @@ -67,11 +59,12 @@ struct Reader( return 0, io.EOF self.prev_rune = -1 - var bytes_written = copy(dest, self.string.as_bytes_slice()[self.read_pos :]) + var bytes_to_read = self.string.as_bytes_slice()[self.read_pos :] + var bytes_written = copy(dest.unsafe_ptr(), bytes_to_read.unsafe_ptr(), len(bytes_to_read)) + dest._len += bytes_written self.read_pos += bytes_written return bytes_written, Error() - @always_inline fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): """Reads from the underlying string into the provided List[UInt8] object. Implements the [io.Reader] trait. @@ -91,8 +84,7 @@ struct Reader( return bytes_read, err - @always_inline - fn _read_at(self, inout dest: Span[UInt8, True], off: Int, capacity: Int) -> (Int, Error): + fn _read_at(self, inout dest: Span[UInt8], off: Int, capacity: Int) -> (Int, Error): """Reads from the Reader into the dest List[UInt8] starting at the offset off. It returns the number of bytes read into dest and an error if any. @@ -112,13 +104,14 @@ struct Reader( return 0, io.EOF var error = Error() - var copied_elements_count = copy(dest, self.string.as_bytes_slice()[off:]) + var bytes_to_read = self.string.as_bytes_slice()[off:] + var copied_elements_count = copy(dest.unsafe_ptr(), bytes_to_read.unsafe_ptr(), len(bytes_to_read)) + dest._len += copied_elements_count if copied_elements_count < len(dest): error = Error(str(io.EOF)) return copied_elements_count, error - @always_inline fn read_at(self, inout dest: List[UInt8], off: Int) -> (Int, Error): """Reads from the Reader into the dest List[UInt8] starting at the offset off. It returns the number of bytes read into dest and an error if any. @@ -139,7 +132,6 @@ struct Reader( return bytes_read, err - @always_inline fn read_byte(inout self) -> (UInt8, Error): """Reads the next byte from the underlying string.""" self.prev_rune = -1 @@ -150,7 +142,6 @@ struct Reader( self.read_pos += 1 return UInt8(b), Error() - @always_inline fn unread_byte(inout self) -> Error: """Unreads the last byte read. Only the most recent byte read can be unread.""" if self.read_pos <= 0: @@ -269,7 +260,6 @@ struct Reader( # return Int(bytes_written) - @always_inline fn reset(inout self, string: String): """Resets the [Reader] to be reading from the beginning of the provided string. diff --git a/gojo/unicode/__init__.mojo b/gojo/unicode/__init__.mojo index b300770..109ae2d 100644 --- a/gojo/unicode/__init__.mojo +++ b/gojo/unicode/__init__.mojo @@ -1 +1 @@ -from .utf8 import rune_count_in_string, UnicodeString, rune_width, string_width, Condition, DEFAULT_CONDITION +from .utf8 import rune_count_in_string, rune_width, string_width, Condition, DEFAULT_CONDITION diff --git a/gojo/unicode/utf8/__init__.mojo b/gojo/unicode/utf8/__init__.mojo index 6f3d8a9..f905a8f 100644 --- a/gojo/unicode/utf8/__init__.mojo +++ b/gojo/unicode/utf8/__init__.mojo @@ -2,5 +2,4 @@ This would not be possible without his help. """ from .runes import rune_count_in_string -from .string import UnicodeString from .width import string_width, rune_width, Condition, DEFAULT_CONDITION diff --git a/gojo/unicode/utf8/runes.mojo b/gojo/unicode/utf8/runes.mojo index 5171282..7681570 100644 --- a/gojo/unicode/utf8/runes.mojo +++ b/gojo/unicode/utf8/runes.mojo @@ -2,14 +2,11 @@ This would not be possible without his help. """ -from ...builtins import Rune from algorithm.functional import vectorize from sys.info import simdwidthof -from bit import countl_zero -# alias simd_width_u8 = simdwidthof[DType.uint8]() -alias simd_width_u8 = 1 +alias simd_width_u8 = simdwidthof[DType.uint8]() fn rune_count_in_string(s: String) -> Int: @@ -21,13 +18,13 @@ fn rune_count_in_string(s: String) -> Int: Returns: The number of runes in the string. """ - var p = DTypePointer[DType.uint8](s.unsafe_uint8_ptr()) + var p = DTypePointer[DType.uint8](s.unsafe_ptr()) var string_byte_length = len(s) var result = 0 @parameter fn count[simd_width: Int](offset: Int): - result += int(((p.load[width=simd_width](offset) >> 6) != 0b10).reduce_add()) + result += int(((SIMD[size=simd_width].load(p + offset) >> 6) != 0b10).cast[DType.uint8]().reduce_add()) vectorize[count, simd_width_u8](string_byte_length) return result diff --git a/gojo/unicode/utf8/string.mojo b/gojo/unicode/utf8/string.mojo deleted file mode 100644 index f0c852d..0000000 --- a/gojo/unicode/utf8/string.mojo +++ /dev/null @@ -1,109 +0,0 @@ -from bit import countl_zero -from algorithm.functional import vectorize -from sys.info import simdwidthof - - -alias simd_width_u8 = simdwidthof[DType.uint8]() - - -@value -struct UnicodeString(Stringable, Sized): - """A string that supports Unicode characters. - - The algorithms to handle UTF-8 are from @maxim on the Mojo Discord. Thanks! - """ - - var inner: String - - @always_inline - fn __init__(inout self, owned s: String): - self.inner = s^ - - @always_inline - fn __init__(inout self, owned bytes: List[UInt8]): - if bytes[-1] != 0: - bytes.append(0) - self.inner = String(bytes^) - - @always_inline - fn __len__(self) -> Int: - """Count the number of runes in a string. - - Returns: - The number of runes in the string. - """ - var data = DTypePointer[DType.uint8](self.inner.unsafe_uint8_ptr()) - var byte_count = len(self.inner) - var result = 0 - - @parameter - fn count[simd_width: Int](offset: Int): - result += int(((data.load[width=simd_width](offset) >> 6) != 0b10).cast[DType.uint8]().reduce_add()) - - vectorize[count, simd_width_u8](byte_count) - return result - - @always_inline - fn __str__(self) -> String: - return self.inner - - @always_inline - fn __getitem__(self: Reference[Self], slice: Slice) -> StringSlice[self.is_mutable, self.lifetime]: - """TODO: Doesn't handle negative indices.""" - var bytes_left = len(self[].inner) - var total_char_length: Int = 0 - for _ in range(slice.start, slice.end): - # Number of bytes of the current character - var char_length = int( - (DTypePointer[DType.uint8](self[].inner.unsafe_uint8_ptr() + total_char_length).load() >> 7 == 0).cast[ - DType.uint8 - ]() - * 1 - + countl_zero(~DTypePointer[DType.uint8](self[].inner.unsafe_uint8_ptr() + total_char_length).load()) - ) - - # Move iterator forward - bytes_left -= char_length - total_char_length += char_length - - return StringSlice[self.is_mutable, self.lifetime]( - unsafe_from_utf8_ptr=self[].inner.unsafe_uint8_ptr(), len=total_char_length - ) - - @always_inline - fn bytecount(self) -> Int: - return len(self.inner) - - @always_inline - fn __iter__( - self: Reference[Self], - ) -> _StringIter[self.is_mutable, self.lifetime]: - return _StringIter(self[].inner) - - -@value -struct _StringIter[mutability: Bool, lifetime: AnyLifetime[mutability].type](): - var bytes_left: Int - var ptr: UnsafePointer[UInt8] - - @always_inline - fn __init__(inout self, src: Reference[String, mutability, lifetime]): - self.bytes_left = len(src[]) - self.ptr = src[].unsafe_uint8_ptr() - - fn __next__(inout self) -> StringSlice[mutability, lifetime]: - # Number of bytes of the current character - var char_length = int( - (DTypePointer[DType.uint8](self.ptr).load() >> 7 == 0).cast[DType.uint8]() * 1 - + countl_zero(~DTypePointer[DType.uint8](self.ptr).load()) - ) - - # Move iterator forward - self.bytes_left -= char_length - self.ptr += char_length - - return StringSlice[mutability, lifetime](unsafe_from_utf8_ptr=self.ptr - char_length, len=char_length) - - @always_inline - fn __len__(self) -> Int: - return self.bytes_left diff --git a/gojo/unicode/utf8/width.mojo b/gojo/unicode/utf8/width.mojo index 8a09ed7..5678d6e 100644 --- a/gojo/unicode/utf8/width.mojo +++ b/gojo/unicode/utf8/width.mojo @@ -1,5 +1,4 @@ from .table import Interval, narrow, combining, doublewidth, ambiguous, emoji, nonprint -from .string import UnicodeString @value @@ -48,7 +47,7 @@ struct Condition: fn string_width(self, s: String) -> Int: """Return width as you can see.""" var width = 0 - for r in UnicodeString(s): + for r in s: width += self.rune_width(ord(String(r))) return width diff --git a/tests/test_bufio.mojo b/tests/test_bufio.mojo index 8d02f3b..debf12c 100644 --- a/tests/test_bufio.mojo +++ b/tests/test_bufio.mojo @@ -22,16 +22,16 @@ fn test_read(): test.assert_equal(to_string(dest), "Hello World!") -fn test_read_all(): - var test = MojoTest("Testing bufio.Reader with io.read_all") +# fn test_read_all(): +# var test = MojoTest("Testing bufio.Reader with io.read_all") - var s: String = "0123456789" - var buf = buffer.new_reader(s) - var reader = Reader(buf^) - var result = read_all(reader) - var bytes = result[0] - bytes.append(0) - test.assert_equal(String(bytes), "0123456789") +# var s: String = "0123456789" +# var buf = buffer.new_reader(s) +# var reader = Reader(buf^) +# var result = read_all(reader) +# var bytes = result[0] +# bytes.append(0) +# test.assert_equal(String(bytes), "0123456789") # fn test_write_to(): @@ -226,7 +226,7 @@ fn test_read_from(): # TODO: Add big file read/write to make sure buffer usage is correct fn main(): test_read() - test_read_all() + # test_read_all() # test_write_to() test_read_and_unread_byte() test_read_slice() diff --git a/tests/test_bufio_scanner.mojo b/tests/test_bufio_scanner.mojo index 7a035b0..671ef4e 100644 --- a/tests/test_bufio_scanner.mojo +++ b/tests/test_bufio_scanner.mojo @@ -1,21 +1,21 @@ from tests.wrapper import MojoTest from gojo.bytes import buffer from gojo.io import FileWrapper -from gojo.bufio import Reader, Scanner, scan_words, scan_bytes, scan_runes +from gojo.bufio import Reader, Scanner, scan_words, scan_bytes fn test_scan_words(): var test = MojoTest("Testing bufio.scan_words") # Create a reader from a string buffer - var s: String = "Testing this string!" + var s: String = "Testing🔥 this🔥 string🔥!" var buf = buffer.new_buffer(s) - var r = Reader(buf^) + # var r = Reader(buf^) # Create a scanner from the reader - var scanner = Scanner[split=scan_words](r^) + var scanner = Scanner[split=scan_words](buf^) - var expected_results = List[String]("Testing", "this", "string!") + var expected_results = List[String]("Testing🔥", "this🔥", "string🔥!") var i = 0 while scanner.scan(): @@ -29,10 +29,10 @@ fn test_scan_lines(): # Create a reader from a string buffer var s: String = "Testing\nthis\nstring!" var buf = buffer.new_buffer(s) - var r = Reader(buf^) + # var r = Reader(buf^) # Create a scanner from the reader - var scanner = Scanner(r^) + var scanner = Scanner(buf^) var expected_results = List[String]("Testing", "this", "string!") var i = 0 @@ -45,10 +45,10 @@ fn test_scan_lines(): fn scan_no_newline_test(test_case: String, result_lines: List[String], test: MojoTest): # Create a reader from a string buffer var buf = buffer.new_buffer(test_case) - var r = Reader(buf^) + # var r = Reader(buf^) # Create a scanner from the reader - var scanner = Scanner(r^) + var scanner = Scanner(buf^) var i = 0 while scanner.scan(): test.assert_equal(scanner.current_token(), result_lines[i]) @@ -92,18 +92,17 @@ fn test_scan_bytes(): var test_cases = List[String]("", "a", "abc", "abc def\n\t\tgh ") - for i in range(len(test_cases)): - var test_case = test_cases[i] + for test_case in test_cases: # Create a reader from a string buffer - var buf = buffer.new_buffer(test_case) - var reader = Reader(buf^) + var buf = buffer.new_buffer(test_case[]) + # var reader = Reader(buf^) # Create a scanner from the reader - var scanner = Scanner[split=scan_bytes](reader^) + var scanner = Scanner[split=scan_bytes](buf^) var j = 0 while scanner.scan(): - test.assert_equal(scanner.current_token(), test_case[j]) + test.assert_equal(scanner.current_token(), test_case[][j]) j += 1 @@ -121,23 +120,23 @@ fn test_file_wrapper_scanner() raises: i += 1 -fn test_scan_runes(): - var test = MojoTest("Testing bufio.scan_runes") +# fn test_scan_runes(): +# var test = MojoTest("Testing bufio.scan_runes") - # Create a reader from a string buffer - var s: String = "🔪🔥🔪" - var buf = buffer.new_buffer(s) - var r = Reader(buf^) +# # Create a reader from a string buffer +# var s: String = "🔪🔥🔪🔥" +# var buf = buffer.new_buffer(s) +# # var r = Reader(buf^) - # Create a scanner from the reader - var scanner = Scanner[split=scan_runes](r^) +# # Create a scanner from the reader +# var scanner = Scanner[split=scan_runes](buf^) - var expected_results = List[String]("🔪", "🔥", "🔪") - var i = 0 +# var expected_results = List[String]("🔪", "🔥", "🔪", "🔥") +# var i = 0 - while scanner.scan(): - test.assert_equal(scanner.current_token(), expected_results[i]) - i += 1 +# while scanner.scan(): +# test.assert_equal(scanner.current_token(), expected_results[i]) +# i += 1 fn main() raises: @@ -149,4 +148,4 @@ fn main() raises: test_scan_lines_cr_empty_final_line() test_scan_bytes() test_file_wrapper_scanner() - test_scan_runes() + # test_scan_runes() diff --git a/tests/test_builtins_bytes.mojo b/tests/test_builtins_bytes.mojo index ae8e851..e84941c 100644 --- a/tests/test_builtins_bytes.mojo +++ b/tests/test_builtins_bytes.mojo @@ -1,6 +1,5 @@ from tests.wrapper import MojoTest -from testing import testing -from gojo.builtins.bytes import Byte, index_byte +from gojo.builtins.bytes import index_byte fn test_index_byte(): @@ -9,16 +8,5 @@ fn test_index_byte(): test.assert_equal(index_byte(bytes, ord("\n")), 5) -fn test_size_and_len(): - var test = MojoTest("Testing builtins.List[Byte].size and builtins.List[Byte].__len__") - var bytes = List[Byte](capacity=16) - - # Size is the number of bytes used, len is the number of bytes allocated. - test.assert_equal(bytes.capacity, 16) - test.assert_equal(len(bytes), 0) - - fn main(): - # test_slice_out_of_bounds() test_index_byte() - test_size_and_len() diff --git a/tests/test_bytes_buffer.mojo b/tests/test_bytes_buffer.mojo index 75e0bc2..0431c71 100644 --- a/tests/test_bytes_buffer.mojo +++ b/tests/test_bytes_buffer.mojo @@ -3,7 +3,7 @@ from gojo.bytes import new_buffer from gojo.bytes.buffer import Buffer -fn test_read() raises: +fn test_read(): var test = MojoTest("Testing bytes.Buffer.read") var s: String = "Hello World!" var buf = new_buffer(s) @@ -13,7 +13,7 @@ fn test_read() raises: test.assert_equal(String(dest), s) -fn test_read_byte() raises: +fn test_read_byte(): var test = MojoTest("Testing bytes.Buffer.read_byte") var s: String = "Hello World!" var buf = new_buffer(s) @@ -21,7 +21,7 @@ fn test_read_byte() raises: test.assert_equal(int(result[0]), 72) -fn test_unread_byte() raises: +fn test_unread_byte(): var test = MojoTest("Testing bytes.Buffer.unread_byte") var s: String = "Hello World!" var buf = new_buffer(s) @@ -33,7 +33,7 @@ fn test_unread_byte() raises: test.assert_equal(buf.offset, 0) -fn test_read_bytes() raises: +fn test_read_bytes(): var test = MojoTest("Testing bytes.Buffer.read_bytes") var s: String = "Hello World!" var buf = new_buffer(s) @@ -43,7 +43,7 @@ fn test_read_bytes() raises: test.assert_equal(String(text), String("Hello")) -fn test_read_slice() raises: +fn test_read_slice(): var test = MojoTest("Testing bytes.Buffer.read_slice") var s: String = "Hello World!" var buf = new_buffer(s) @@ -53,7 +53,7 @@ fn test_read_slice() raises: test.assert_equal(String(text), String("Hello")) -fn test_read_string() raises: +fn test_read_string(): var test = MojoTest("Testing bytes.Buffer.read_string") var s: String = "Hello World!" var buf = new_buffer(s) @@ -61,7 +61,7 @@ fn test_read_string() raises: test.assert_equal(String(result[0]), String("Hello")) -fn test_next() raises: +fn test_next(): var test = MojoTest("Testing bytes.Buffer.next") var buf = new_buffer("Hello World!") var text = List[UInt8](buf.next(5)) @@ -69,7 +69,7 @@ fn test_next() raises: test.assert_equal(String(text), String("Hello")) -fn test_write() raises: +fn test_write(): var test = MojoTest("Testing bytes.Buffer.write") var b = List[UInt8](capacity=16) var buf = new_buffer(b^) @@ -77,7 +77,7 @@ fn test_write() raises: test.assert_equal(str(buf), String("Hello World!")) -fn test_write_string() raises: +fn test_write_string(): var test = MojoTest("Testing bytes.Buffer.write_string") var b = List[UInt8](capacity=16) var buf = new_buffer(b^) @@ -86,7 +86,7 @@ fn test_write_string() raises: test.assert_equal(str(buf), String("\nGoodbye World!")) -fn test_write_byte() raises: +fn test_write_byte(): var test = MojoTest("Testing bytes.Buffer.write_byte") var b = List[UInt8](capacity=16) var buf = new_buffer(b^) @@ -94,7 +94,7 @@ fn test_write_byte() raises: test.assert_equal(str(buf), String("A")) -fn test_new_buffer() raises: +fn test_new_buffer(): var test = MojoTest("Testing bytes.new_buffer") var b = String("Hello World!").as_bytes() var buf = new_buffer(b^) @@ -107,7 +107,7 @@ fn test_new_buffer() raises: test.assert_equal(str(buf), "") -fn main() raises: +fn main(): test_read() test_read_byte() test_unread_byte() diff --git a/tests/test_fmt.mojo b/tests/test_fmt.mojo index a7eaa21..b1b0b7a 100644 --- a/tests/test_fmt.mojo +++ b/tests/test_fmt.mojo @@ -16,9 +16,6 @@ fn test_sprintf() raises: "Hello, world. I am 29 years old. More precisely, I am 29.5 years old. It is True that I like Mojo!", ) - s = sprintf("This is a number: %d. In base 16: %x. In base 16 upper: %X.", 42, 42, 42) - test.assert_equal(s, "This is a number: 42. In base 16: 2a. In base 16 upper: 2A.") - s = sprintf("Hello %s", String("world").as_bytes()) test.assert_equal(s, "Hello world") diff --git a/tests/test_get_addr.mojo b/tests/test_get_addr.mojo index 2f481d2..21e5776 100644 --- a/tests/test_get_addr.mojo +++ b/tests/test_get_addr.mojo @@ -40,7 +40,7 @@ fn test_listener() raises: var listener = listen_tcp("tcp", TCPAddr("0.0.0.0", 8081)) while True: var conn = listener.accept() - print("Accepted connection from", conn.remote_address()) + print("Accepted connection from", str(conn.remote_address())) var err = conn.close() if err: raise err diff --git a/tests/test_std.mojo b/tests/test_std.mojo index fa5aaf5..2484277 100644 --- a/tests/test_std.mojo +++ b/tests/test_std.mojo @@ -3,11 +3,11 @@ from gojo.syscall import FD from gojo.io import STDWriter -fn test_writer() raises: +fn test_writer(): var test = MojoTest("Testing STDWriter.write") var writer = STDWriter[FD.STDOUT]() _ = writer.write_string("") -fn main() raises: +fn main(): test_writer() diff --git a/tests/test_strings_reader.mojo b/tests/test_strings_reader.mojo index feb5be0..cb6321a 100644 --- a/tests/test_strings_reader.mojo +++ b/tests/test_strings_reader.mojo @@ -3,7 +3,7 @@ from gojo.strings import StringBuilder, Reader, new_reader import gojo.io -fn test_read() raises: +fn test_read(): var test = MojoTest("Testing strings.Reader.read") var example: String = "Hello, World!" var reader = new_reader("Hello, World!") @@ -17,7 +17,7 @@ fn test_read() raises: test.assert_equal(String(buffer), "Hello, World!") -fn test_read_slice() raises: +fn test_read_slice(): var test = MojoTest("Testing strings.Reader.read") var example: String = "Hello, World!" var reader = new_reader("Hello, World!") @@ -31,7 +31,7 @@ fn test_read_slice() raises: test.assert_equal(String(buffer), "Hello, World!") -fn test_read_at() raises: +fn test_read_at(): var test = MojoTest("Testing strings.Reader.read_at") var example: String = "Hello, World!" var reader = new_reader("Hello, World!") @@ -45,7 +45,7 @@ fn test_read_at() raises: test.assert_equal(String(buffer), "World!") -fn test_seek() raises: +fn test_seek(): var test = MojoTest("Testing strings.Reader.seek") var reader = new_reader("Hello, World!") @@ -54,7 +54,7 @@ fn test_seek() raises: test.assert_equal(int(position[0]), 5) -fn test_read_and_unread_byte() raises: +fn test_read_and_unread_byte(): var test = MojoTest("Testing strings.Reader.read_byte and strings.Reader.unread_byte") var example: String = "Hello, World!" var reader = new_reader("Hello, World!") @@ -68,7 +68,7 @@ fn test_read_and_unread_byte() raises: test.assert_equal(len(reader), len(example)) -# fn test_write_to() raises: +# fn test_write_to(): # var test = MojoTest("Testing strings.Reader.write_to") # var example: String = "Hello, World!" # var reader = new_reader("Hello, World!") @@ -79,7 +79,7 @@ fn test_read_and_unread_byte() raises: # test.assert_equal(str(builder), example) -fn main() raises: +fn main(): test_read() test_read_at() test_seek() diff --git a/tests/test_unicode_string.mojo b/tests/test_unicode_string.mojo deleted file mode 100644 index c434bcd..0000000 --- a/tests/test_unicode_string.mojo +++ /dev/null @@ -1,23 +0,0 @@ -from gojo.unicode import UnicodeString -from tests.wrapper import MojoTest - - -fn test_unicode_string(): - var test = MojoTest("Testing unicode.UnicodeString") - var s = UnicodeString("𡨸漢𡨸漢") - test.assert_equal(s.bytecount(), 14) - test.assert_equal(len(s), 4) - - var i = 0 - var results = List[String]("𡨸", "漢", "𡨸", "漢") - for c in s: - test.assert_equal(String(c), results[i]) - i += 1 - - test.assert_equal(String(s[:1]), "𡨸") - test.assert_equal(String(s[:2]), "𡨸漢") - # test.assert_equal(String(s[:-1]), "𡨸漢𡨸漢") - - -fn main(): - test_unicode_string() diff --git a/tests/test_unicode_width.mojo b/tests/test_unicode_width.mojo index c223038..4afa073 100644 --- a/tests/test_unicode_width.mojo +++ b/tests/test_unicode_width.mojo @@ -1,18 +1,18 @@ -from gojo.unicode import string_width, rune_width, UnicodeString +from gojo.unicode import string_width, rune_width from tests.wrapper import MojoTest fn test_string_width(): var test = MojoTest("Testing unicode.string_width and unicode.rune_width") - var ascii = "Hello, World!" + var ascii: String = "Hello, World!" var s: String = "𡨸漢𡨸漢" test.assert_equal(string_width(s), 8) test.assert_equal(string_width(ascii), 13) - for r in UnicodeString(s): + for r in s: test.assert_equal(rune_width(ord(String(r))), 2) - for r in UnicodeString(ascii): + for r in ascii: test.assert_equal(rune_width(ord(String(r))), 1)