fix utf8 handling when importing json (#14168)

oven-sh · Sep 26, 2024 · af12ff1 · af12ff1
1 parent 80db770
commit af12ff1
Show file tree

Hide file tree

Showing 10 changed files with 71 additions and 18 deletions.
diff --git a/src/StandaloneModuleGraph.zig b/src/StandaloneModuleGraph.zig
@@ -999,7 +999,7 @@ pub const StandaloneModuleGraph = struct {
             bun.JSAst.Expr.Data.Store.reset();
             bun.JSAst.Stmt.Data.Store.reset();
         }
-        var json = bun.JSON.ParseJSON(&json_src, &log, arena) catch
+        var json = bun.JSON.ParseJSON(&json_src, &log, arena, false) catch
             return error.InvalidSourceMap;
 
         const mappings_str = json.get("mappings") orelse

diff --git a/src/bundler.zig b/src/bundler.zig
@@ -1473,9 +1473,9 @@ pub const Bundler = struct {
                     // We allow importing tsconfig.*.json or jsconfig.*.json with comments
                     // These files implicitly become JSONC files, which aligns with the behavior of text editors.
                     if (source.path.isJSONCFile())
-                        json_parser.ParseTSConfig(&source, bundler.log, allocator) catch return null
+                        json_parser.ParseTSConfig(&source, bundler.log, allocator, false) catch return null
                     else
-                        json_parser.ParseJSON(&source, bundler.log, allocator) catch return null
+                        json_parser.ParseJSON(&source, bundler.log, allocator, false) catch return null
                 else if (kind == .toml)
                     TOML.parse(&source, bundler.log, allocator) catch return null
                 else

diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig
@@ -2736,7 +2736,7 @@ pub const ParseTask = struct {
             .json => {
                 const trace = tracer(@src(), "ParseJSON");
                 defer trace.end();
-                const root = (try resolver.caches.json.parsePackageJSON(log, source, allocator)) orelse Expr.init(E.Object, E.Object{}, Logger.Loc.Empty);
+                const root = (try resolver.caches.json.parsePackageJSON(log, source, allocator, false)) orelse Expr.init(E.Object, E.Object{}, Logger.Loc.Empty);
                 return JSAst.init((try js_parser.newLazyExportAST(allocator, bundler.options.define, opts, log, root, &source, "")).?);
             },
             .toml => {

diff --git a/src/bunfig.zig b/src/bunfig.zig
@@ -806,7 +806,7 @@ pub const Bunfig = struct {
                 ctx.log.addErrorFmt(&source, logger.Loc.Empty, allocator, "Failed to parse", .{}) catch unreachable;
             }
             return err;
-        } else JSONParser.ParseTSConfig(&source, ctx.log, allocator) catch |err| {
+        } else JSONParser.ParseTSConfig(&source, ctx.log, allocator, true) catch |err| {
             if (ctx.log.errors + ctx.log.warnings == log_count) {
                 ctx.log.addErrorFmt(&source, logger.Loc.Empty, allocator, "Failed to parse", .{}) catch unreachable;
             }

diff --git a/src/cache.zig b/src/cache.zig
@@ -294,12 +294,12 @@ pub const Json = struct {
     pub fn init(_: std.mem.Allocator) Json {
         return Json{};
     }
-    fn parse(_: *@This(), log: *logger.Log, source: logger.Source, allocator: std.mem.Allocator, comptime func: anytype) anyerror!?js_ast.Expr {
+    fn parse(_: *@This(), log: *logger.Log, source: logger.Source, allocator: std.mem.Allocator, comptime func: anytype, comptime force_utf8: bool) anyerror!?js_ast.Expr {
         var temp_log = logger.Log.init(allocator);
         defer {
             temp_log.appendToMaybeRecycled(log, &source) catch {};
         }
-        return func(&source, &temp_log, allocator) catch handler: {
+        return func(&source, &temp_log, allocator, force_utf8) catch handler: {
             break :handler null;
         };
     }
@@ -308,17 +308,17 @@ pub const Json = struct {
         // They are JSON files with comments and trailing commas.
         // Sometimes tooling expects this to work.
         if (source.path.isJSONCFile()) {
-            return try parse(cache, log, source, allocator, json_parser.ParseTSConfig);
+            return try parse(cache, log, source, allocator, json_parser.ParseTSConfig, true);
         }
 
-        return try parse(cache, log, source, allocator, json_parser.ParseJSON);
+        return try parse(cache, log, source, allocator, json_parser.ParseJSON, false);
     }
 
-    pub fn parsePackageJSON(cache: *@This(), log: *logger.Log, source: logger.Source, allocator: std.mem.Allocator) anyerror!?js_ast.Expr {
-        return try parse(cache, log, source, allocator, json_parser.ParseTSConfig);
+    pub fn parsePackageJSON(cache: *@This(), log: *logger.Log, source: logger.Source, allocator: std.mem.Allocator, comptime force_utf8: bool) anyerror!?js_ast.Expr {
+        return try parse(cache, log, source, allocator, json_parser.ParseTSConfig, force_utf8);
     }
 
     pub fn parseTSConfig(cache: *@This(), log: *logger.Log, source: logger.Source, allocator: std.mem.Allocator) anyerror!?js_ast.Expr {
-        return try parse(cache, log, source, allocator, json_parser.ParseTSConfig);
+        return try parse(cache, log, source, allocator, json_parser.ParseTSConfig, true);
     }
 };
diff --git a/src/json_parser.zig b/src/json_parser.zig
@@ -714,6 +714,7 @@ pub fn ParseJSON(
     source: *const logger.Source,
     log: *logger.Log,
     allocator: std.mem.Allocator,
+    comptime force_utf8: bool,
 ) !Expr {
     var parser = try JSONParser.init(allocator, source.*, log);
     switch (source.contents.len) {
@@ -734,7 +735,7 @@ pub fn ParseJSON(
         else => {},
     }
 
-    return try parser.parseExpr(false, false);
+    return try parser.parseExpr(false, force_utf8);
 }
 
 /// Parse Package JSON
@@ -1023,7 +1024,7 @@ pub fn ParseEnvJSON(source: *const logger.Source, log: *logger.Log, allocator: s
     }
 }
 
-pub fn ParseTSConfig(source: *const logger.Source, log: *logger.Log, allocator: std.mem.Allocator) !Expr {
+pub fn ParseTSConfig(source: *const logger.Source, log: *logger.Log, allocator: std.mem.Allocator, comptime force_utf8: bool) !Expr {
     switch (source.contents.len) {
         // This is to be consisntent with how disabled JS files are handled
         0 => {
@@ -1044,7 +1045,7 @@ pub fn ParseTSConfig(source: *const logger.Source, log: *logger.Log, allocator:
 
     var parser = try TSConfigParser.init(allocator, source.*, log);
 
-    return parser.parseExpr(false, true);
+    return parser.parseExpr(false, force_utf8);
 }
 
 const duplicateKeyJson = "{ \"name\": \"valid\", \"name\": \"invalid\" }";

diff --git a/src/resolver/package_json.zig b/src/resolver/package_json.zig
@@ -641,7 +641,7 @@ pub const PackageJSON = struct {
         var json_source = logger.Source.initPathString(key_path.text, entry.contents);
         json_source.path.pretty = r.prettyPath(json_source.path);
 
-        const json: js_ast.Expr = (r.caches.json.parsePackageJSON(r.log, json_source, allocator) catch |err| {
+        const json: js_ast.Expr = (r.caches.json.parsePackageJSON(r.log, json_source, allocator, true) catch |err| {
             if (Environment.isDebug) {
                 Output.printError("{s}: JSON parse error: {s}", .{ package_json_path, @errorName(err) });
             }

diff --git a/src/resolver/resolver.zig b/src/resolver/resolver.zig
@@ -734,7 +734,7 @@ pub const Resolver = struct {
         // support passing a package.json or path to a package
         const pkg: *const PackageJSON = result.package_json orelse r.packageJSONForResolvedNodeModuleWithIgnoreMissingName(&result, true) orelse return error.MissingPackageJSON;
 
-        const json = (try r.caches.json.parsePackageJSON(r.log, pkg.source, r.allocator)) orelse return error.JSONParseError;
+        const json = (try r.caches.json.parsePackageJSON(r.log, pkg.source, r.allocator, true)) orelse return error.JSONParseError;
 
         pkg.loadFrameworkWithPreference(pair, json, r.allocator, load_defines, preference);
         const dir = pkg.source.path.sourceDir();

diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig
@@ -132,7 +132,7 @@ pub fn parseJSON(
         bun.JSAst.Stmt.Data.Store.reset();
     }
     debug("parse (JSON, {d} bytes)", .{source.len});
-    var json = bun.JSON.ParseJSON(&json_src, &log, arena) catch {
+    var json = bun.JSON.ParseJSON(&json_src, &log, arena, false) catch {
         return error.InvalidJSON;
     };
 

diff --git a/test/bundler/bundler_edgecase.test.ts b/test/bundler/bundler_edgecase.test.ts
@@ -1831,6 +1831,58 @@ describe("bundler", () => {
     },
     run: { stdout: "1\n2" },
   });
+  itBundled("edgecase/Latin1StringInImportedJSON", {
+    files: {
+      "/entry.ts": `
+        import x from './second.json';
+        console.log(x + 'a');
+      `,
+      "/second.json": `
+        "测试"
+      `,
+    },
+    target: "bun",
+    run: { stdout: `测试a` },
+  });
+  itBundled("edgecase/Latin1StringInImportedJSONBrowser", {
+    files: {
+      "/entry.ts": `
+        import x from './second.json';
+        console.log(x + 'a');
+      `,
+      "/second.json": `
+        "测试"
+      `,
+    },
+    target: "browser",
+    run: { stdout: `测试a` },
+  });
+  itBundled("edgecase/Latin1StringKey", {
+    files: {
+      "/entry.ts": `
+        import x from './second.json';
+        console.log(x["测试" + "a"]);
+      `,
+      "/second.json": `
+        {"测试a" : 123}
+      `,
+    },
+    target: "bun",
+    run: { stdout: `123` },
+  });
+  itBundled("edgecase/Latin1StringKeyBrowser", {
+    files: {
+      "/entry.ts": `
+        import x from './second.json';
+        console.log(x["测试" + "a"]);
+      `,
+      "/second.json": `
+        {"测试a" : 123}
+      `,
+    },
+    target: "browser",
+    run: { stdout: `123` },
+  });
 
   // TODO(@paperdave): test every case of this. I had already tested it manually, but it may break later
   const requireTranspilationListESM = [