Support UTF-32 Encoding

lib/rexml/document.rb

@@ -284,6 +284,8 @@ def version
     # Returns the XMLDecl encoding of the document,
     # if it has been set, otherwise the default encoding:
     #
+    #   d = REXML::Document.new('<?xml version="1.0" encoding="UTF-32"?>')
+    #   d.encoding # => "UTF-32"
     #   d = REXML::Document.new('<?xml version="1.0" encoding="UTF-16"?>')
     #   d.encoding # => "UTF-16"
     #   d = REXML::Document.new('')

lib/rexml/output.rb

@@ -13,7 +13,10 @@ def initialize real_IO, encd="iso-8859-1"
 
       @to_utf = encoding != 'UTF-8'
 
-      if encoding == "UTF-16"
+      if encoding == "UTF-32"
+        @output << "\ufeff".encode("UTF-32BE")
+        self.encoding = "UTF-32BE"
+      elsif encoding == "UTF-16"
         @output << "\ufeff".encode("UTF-16BE")
         self.encoding = "UTF-16BE"
       end

lib/rexml/parsers/baseparser.rb

@@ -167,6 +167,7 @@ def initialize( source )
         @entity_expansion_count = 0
         @entity_expansion_limit = Security.entity_expansion_limit
         @entity_expansion_text_limit = Security.entity_expansion_text_limit
+        @source.ensure_buffer
       end
 
       def add_listener( listener )
@@ -639,7 +640,7 @@ def record_entity_expansion(delta=1)
 
       def need_source_encoding_update?(xml_declaration_encoding)
         return false if xml_declaration_encoding.nil?
-        return false if /\AUTF-16\z/i =~ xml_declaration_encoding
+        return false if /\AUTF-(32|16)\z/i =~ xml_declaration_encoding
         true
       end
 
@@ -747,8 +748,8 @@ def process_instruction
           if need_source_encoding_update?(encoding)
             @source.encoding = encoding
           end
-          if encoding.nil? and /\AUTF-16(?:BE|LE)\z/i =~ @source.encoding
-            encoding = "UTF-16"
+          if encoding.nil? and /\AUTF-(32|16)(?:BE|LE)\z/i =~ @source.encoding
+            encoding = "UTF-#{$1}"
           end
           standalone = STANDALONE.match(content)
           standalone = standalone[1] unless standalone.nil?

lib/rexml/source.rb

@@ -154,7 +154,11 @@ def detect_encoding
       detected_encoding = "UTF-8"
       begin
         @scanner.string.force_encoding("ASCII-8BIT")
-        if @scanner.scan(/\xfe\xff/n)
+        if @scanner.scan(/\x00\x00\xfe\xff/n)
+          detected_encoding = "UTF-32BE"
+        elsif @scanner.scan(/\xff\xfe\x00\x00/n)
+          detected_encoding = "UTF-32LE"
+        elsif @scanner.scan(/\xfe\xff/n)
           detected_encoding = "UTF-16BE"
         elsif @scanner.scan(/\xff\xfe/n)
           detected_encoding = "UTF-16LE"
@@ -192,7 +196,7 @@ def initialize(arg, block_size=500, encoding=nil)
       if encoding
         super("", encoding)
       else
-        super(@source.read(3) || "")
+        super(@source.read(4) || "")
       end
 
       if !@to_utf and
@@ -295,14 +299,19 @@ def current_line
 
     private
     def readline(term = nil)
-      str = @source.readline(term || @line_break)
       if @pending_buffer
+        begin
+          str = @source.readline(term || @line_break)
+        rescue IOError
+        end
         if str.nil?
           str = @pending_buffer
         else
           str = @pending_buffer + str
         end
         @pending_buffer = nil
+      else
+        str = @source.readline(term || @line_break)
       end
       return nil if str.nil?
 
@@ -316,7 +325,7 @@ def readline(term = nil)
 
     def encoding_updated
       case @encoding
-      when "UTF-16BE", "UTF-16LE"
+      when "UTF-32BE", "UTF-32LE", "UTF-16BE", "UTF-16LE"
         @source.binmode
         @source.set_encoding(@encoding, @encoding)
       end

test/parse/test_text.rb

@@ -4,6 +4,23 @@
 module REXMLTests
   class TestParseText < Test::Unit::TestCase
     class TestInvalid < self
+      def test_text_only
+        exception = assert_raise(REXML::ParseException) do
+          parser = REXML::Parsers::BaseParser.new('a')
+          while parser.has_next?
+            parser.pull
+          end
+        end
+
+        assert_equal(<<~DETAIL.chomp, exception.to_s)
+          Malformed XML: Content at the start of the document (got 'a')
+          Line: 1
+          Position: 1
+          Last 80 unconsumed characters:
+
+        DETAIL
+      end
+
       def test_before_root
         exception = assert_raise(REXML::ParseException) do
           parser = REXML::Parsers::BaseParser.new('b<a></a>')
@@ -15,9 +32,9 @@ def test_before_root
         assert_equal(<<~DETAIL.chomp, exception.to_s)
           Malformed XML: Content at the start of the document (got 'b')
           Line: 1
-          Position: 4
+          Position: 8
           Last 80 unconsumed characters:
-          <a>
+          <a></a>
         DETAIL
       end
 

test/test_core.rb

@@ -488,6 +488,11 @@ def test_xmldecl_utf_16be_encoding_name
                    XMLDecl.new("1.0", "UTF-16").to_s)
     end
 
+    def test_xmldecl_utf_32be_encoding_name
+      assert_equal("<?xml version='1.0' encoding='UTF-32'?>",
+                   XMLDecl.new("1.0", "UTF-32").to_s)
+    end
+
     def each_test( element, xpath, num_children )
       count = 0
       element.each_element( xpath ) { |child|

test/test_document.rb

@@ -351,6 +351,26 @@ def test_utf_16be
           document = REXML::Document.new(bom + xml)
           assert_equal("UTF-16", document.encoding)
         end
+
+        def test_utf_32le
+          xml = <<-EOX.encode("UTF-32LE").force_encoding("ASCII-8BIT")
+<?xml version="1.0" encoding="UTF-32"?>
+<message>Hello world!</message>
+EOX
+          bom = "\ufeff".encode("UTF-32LE").force_encoding("ASCII-8BIT")
+          document = REXML::Document.new(bom + xml)
+          assert_equal("UTF-32", document.encoding)
+        end
+
+        def test_utf_32be
+          xml = <<-EOX.encode("UTF-32BE").force_encoding("ASCII-8BIT")
+<?xml version="1.0" encoding="UTF-32"?>
+<message>Hello world!</message>
+EOX
+          bom = "\ufeff".encode("UTF-32BE").force_encoding("ASCII-8BIT")
+          document = REXML::Document.new(bom + xml)
+          assert_equal("UTF-32", document.encoding)
+        end
       end
 
       class NoEncodingTest < self
@@ -383,6 +403,26 @@ def test_utf_16be
           document = REXML::Document.new(bom + xml)
           assert_equal("UTF-16", document.encoding)
         end
+
+        def test_utf_32le
+          xml = <<-EOX.encode("UTF-32LE").force_encoding("ASCII-8BIT")
+<?xml version="1.0"?>
+<message>Hello world!</message>
+EOX
+          bom = "\ufeff".encode("UTF-32LE").force_encoding("ASCII-8BIT")
+          document = REXML::Document.new(bom + xml)
+          assert_equal("UTF-32", document.encoding)
+        end
+
+        def test_utf_32be
+          xml = <<-EOX.encode("UTF-32BE").force_encoding("ASCII-8BIT")
+<?xml version="1.0"?>
+<message>Hello world!</message>
+EOX
+          bom = "\ufeff".encode("UTF-32BE").force_encoding("ASCII-8BIT")
+          document = REXML::Document.new(bom + xml)
+          assert_equal("UTF-32", document.encoding)
+        end
       end
 
       class WriteTest < self
@@ -399,13 +439,30 @@ def test_utf_16
           expected_xml = <<-EOX.chomp.encode("UTF-16BE")
 \ufeff<?xml version='1.0' encoding='UTF-16'?>
 <message>Hello world!</message>
+EOX
+          assert_equal(expected_xml, actual_xml)
+        end
+
+        def test_utf_32
+          xml = <<-EOX.encode("UTF-32LE").force_encoding("ASCII-8BIT")
+<?xml version="1.0"?>
+<message>Hello world!</message>
+EOX
+          bom = "\ufeff".encode("UTF-32LE").force_encoding("ASCII-8BIT")
+          document = REXML::Document.new(bom + xml)
+
+          actual_xml = ""
+          document.write(actual_xml)
+          expected_xml = <<-EOX.chomp.encode("UTF-32BE")
+\ufeff<?xml version='1.0' encoding='UTF-32'?>
+<message>Hello world!</message>
 EOX
           assert_equal(expected_xml, actual_xml)
         end
       end
 
       class ReadUntilTest < Test::Unit::TestCase
-       def test_utf_8
+        def test_utf_8
           xml = <<-EOX.force_encoding("ASCII-8BIT")
 <?xml version="1.0" encoding="UTF-8"?>
 <message testing=">">Hello world!</message>
@@ -436,6 +493,28 @@ def test_utf_16be
           assert_equal("UTF-16", document.encoding)
           assert_equal(">", REXML::XPath.match(document, "/message")[0].attribute("testing").value)
         end
+
+        def test_utf_32le
+          xml = <<-EOX.encode("UTF-32LE").force_encoding("ASCII-8BIT")
+<?xml version="1.0" encoding="UTF-32"?>
+<message testing=">">Hello world!</message>
+EOX
+          bom = "\ufeff".encode("UTF-32LE").force_encoding("ASCII-8BIT")
+          document = REXML::Document.new(bom + xml)
+          assert_equal("UTF-32", document.encoding)
+          assert_equal(">", REXML::XPath.match(document, "/message")[0].attribute("testing").value)
+        end
+
+        def test_utf_32be
+          xml = <<-EOX.encode("UTF-32BE").force_encoding("ASCII-8BIT")
+<?xml version="1.0" encoding="UTF-32"?>
+<message testing=">">Hello world!</message>
+EOX
+          bom = "\ufeff".encode("UTF-32BE").force_encoding("ASCII-8BIT")
+          document = REXML::Document.new(bom + xml)
+          assert_equal("UTF-32", document.encoding)
+          assert_equal(">", REXML::XPath.match(document, "/message")[0].attribute("testing").value)
+        end
       end
     end
   end

test/test_encoding.rb

@@ -92,7 +92,15 @@ def test_parse_utf16
         REXML::Document.new(f)
       end
       assert_equal("UTF-16", utf16.encoding)
-      assert( utf16[0].kind_of?(REXML::XMLDecl))
+      assert(utf16[0].kind_of?(REXML::XMLDecl))
+    end
+
+    def test_parse_utf32
+      utf32 = File.open(fixture_path("utf32.xml")) do |f|
+        REXML::Document.new(f)
+      end
+      assert_equal("UTF-32", utf32.encoding)
+      assert(utf32[0].kind_of?(REXML::XMLDecl))
     end
 
     def test_parse_utf16_with_utf8_default_internal
@@ -103,5 +111,14 @@ def test_parse_utf16_with_utf8_default_internal
         assert_equal("UTF-16", utf16.encoding)
       end
     end
+
+    def test_parse_utf32_with_utf8_default_internal
+      with_default_internal("UTF-8") do
+        utf32 = File.open(fixture_path("utf32.xml")) do |f|
+          REXML::Document.new(f)
+        end
+        assert_equal("UTF-32", utf32.encoding)
+      end
+    end
   end
 end