Convert std/png text from Latin-1 to UTF-8

Fixes #55
This commit is contained in:
Nigel Tao committed 2021-11-17 23:04:49 +11:00
1 parent e0cdaeb54f
commit 29560a336b
5 files changed
+395 -69

No files matched your search

+39
View File
@@ -17,10 +17,12 @@ pub status "#bad checksum"
pub status "#bad chunk"
pub status "#bad filter"
pub status "#bad header"
pub status "#bad text chunk (not Latin-1)"
pub status "#missing palette"
pub status "#unsupported PNG compression method"
pub status "#unsupported PNG file"
pri status "#internal error: inconsistent I/O"
pri status "#internal error: inconsistent chunk type"
pri status "#internal error: inconsistent frame bounds"
pri status "#internal error: inconsistent workbuf length"
@@ -101,3 +103,40 @@ pri const NUM_CHANNELS : array[8] base.u8[..= 4] = [
4, // color_type == 6: RGBA
0,
]
// LATIN_1 converts from printable Latin-1 (ISO/IEC 8859-1) to UTF-8 (encoded
// as little-endian u16). It was created by script/print-png-latin-1.go.
pri const LATIN_1 : array[256] base.u16 = [
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,
0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,
0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F,
0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,
0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0xA1C2, 0xA2C2, 0xA3C2, 0xA4C2, 0xA5C2, 0xA6C2, 0xA7C2,
0xA8C2, 0xA9C2, 0xAAC2, 0xABC2, 0xACC2, 0xADC2, 0xAEC2, 0xAFC2,
0xB0C2, 0xB1C2, 0xB2C2, 0xB3C2, 0xB4C2, 0xB5C2, 0xB6C2, 0xB7C2,
0xB8C2, 0xB9C2, 0xBAC2, 0xBBC2, 0xBCC2, 0xBDC2, 0xBEC2, 0xBFC2,
0x80C3, 0x81C3, 0x82C3, 0x83C3, 0x84C3, 0x85C3, 0x86C3, 0x87C3,
0x88C3, 0x89C3, 0x8AC3, 0x8BC3, 0x8CC3, 0x8DC3, 0x8EC3, 0x8FC3,
0x90C3, 0x91C3, 0x92C3, 0x93C3, 0x94C3, 0x95C3, 0x96C3, 0x97C3,
0x98C3, 0x99C3, 0x9AC3, 0x9BC3, 0x9CC3, 0x9DC3, 0x9EC3, 0x9FC3,
0xA0C3, 0xA1C3, 0xA2C3, 0xA3C3, 0xA4C3, 0xA5C3, 0xA6C3, 0xA7C3,
0xA8C3, 0xA9C3, 0xAAC3, 0xABC3, 0xACC3, 0xADC3, 0xAEC3, 0xAFC3,
0xB0C3, 0xB1C3, 0xB2C3, 0xB3C3, 0xB4C3, 0xB5C3, 0xB6C3, 0xB7C3,
0xB8C3, 0xB9C3, 0xBAC3, 0xBBC3, 0xBCC3, 0xBDC3, 0xBEC3, 0xBFC3,
]
+106 -15
View File
@@ -166,12 +166,25 @@ pub struct decoder? implements base.image_decoder(
metadata_y : base.u64,
metadata_z : base.u64,
// ztxt_ri and ztxt_wi are read and write indexes into the dst_palette
// buffer, re-purposed as a zlib uncompression buffer for zTXt chunks. The
// upper bound, 1024, is the same as the dst_palette length.
ztxt_ri : base.u32[..= 1024],
ztxt_wi : base.u32[..= 1024],
// ztxt_hist_pos is the history position: how many uncompressed bytes have
// been generated.
ztxt_hist_pos : base.u64,
swizzler : base.pixel_swizzler,
util : base.utility,
)(
crc32 : crc32.ieee_hasher,
zlib : zlib.decoder,
// dst_palette and src_palette are used by the swizzler, during
// decode_frame. src_palette is initialized by processing the PLTE chunk.
// dst_palette is also re-purposed as a zlib uncompression buffer for zTXt
// chunks, during decode_image_config.
dst_palette : array[4 * 256] base.u8,
src_palette : array[4 * 256] base.u8,
)
@@ -1324,6 +1337,10 @@ pub func decoder.set_report_metadata!(fourcc: base.u32, report: base.bool) {
pub func decoder.tell_me_more?(dst: base.io_writer, minfo: nptr base.more_information, src: base.io_reader) {
var c : base.u8
var c2 : base.u16
var w : base.io_writer
var num_written : base.u64
var w_mark : base.u64
var r_mark : base.u64
var zlib_status : base.status
@@ -1342,6 +1359,7 @@ pub func decoder.tell_me_more?(dst: base.io_writer, minfo: nptr base.more_inform
}
}
this.zlib_is_dirty = true
this.ztxt_hist_pos = 0
}
while.loop true {
@@ -1395,28 +1413,87 @@ pub func decoder.tell_me_more?(dst: base.io_writer, minfo: nptr base.more_inform
yield? zlib_status
} else if this.chunk_type == 'zTXt'le {
// TODO: Convert uncompressed data from Latin-1 to UTF-8.
io_limit (io: args.src, limit: this.chunk_length as base.u64) {
r_mark = args.src.mark()
zlib_status =? this.zlib.transform_io?(
dst: args.dst, src: args.src, workbuf: this.util.empty_slice_u8())
this.chunk_length ~sat-=
(args.src.count_since(mark: r_mark) & 0xFFFF_FFFF) as base.u32
// Fill this.dst_palette, zlib-uncompressing producing Latin-1.
if this.ztxt_ri == this.ztxt_wi {
io_bind (io: w, data: this.dst_palette[..], history_position: this.ztxt_hist_pos) {
io_limit (io: args.src, limit: this.chunk_length as base.u64) {
w_mark = w.mark()
r_mark = args.src.mark()
zlib_status =? this.zlib.transform_io?(
dst: w, src: args.src, workbuf: this.util.empty_slice_u8())
this.chunk_length ~sat-=
(args.src.count_since(mark: r_mark) & 0xFFFF_FFFF) as base.u32
num_written = w.count_since(mark: w_mark)
}
}
if num_written > 1024 {
return "#internal error: inconsistent I/O"
}
this.ztxt_ri = 0
this.ztxt_wi = num_written as base.u32
this.ztxt_hist_pos ~sat+= num_written
}
// Drain this.dst_palette, converting from Latin-1 to UTF-8.
while this.ztxt_ri < this.ztxt_wi {
assert this.ztxt_ri < 1024 via "a < b: a < c; c <= b"(c: this.ztxt_wi)
c2 = LATIN_1[this.dst_palette[this.ztxt_ri]]
if c2 == 0 {
return "#bad text chunk (not Latin-1)"
} else if c2 <= 0x7F {
if args.dst.length() <= 0 {
yield? base."$short write"
c2 = 0
continue.loop
}
this.ztxt_ri += 1
args.dst.write_u8_fast!(a: c2 as base.u8)
} else {
if args.dst.length() <= 1 {
yield? base."$short write"
c2 = 0
continue.loop
}
this.ztxt_ri += 1
args.dst.write_u16le_fast!(a: c2)
}
} endwhile
if zlib_status.is_ok() {
this.metadata_is_zlib_compressed = false
break.loop
} else if not zlib_status.is_suspension() {
return zlib_status
} else if zlib_status <> base."$short write" {
yield? zlib_status
}
yield? zlib_status
} else {
return "#internal error: inconsistent chunk type"
}
} else if (this.chunk_type == 'iTXt'le) and (this.metadata_fourcc == 'KVPV'be) {
// iTXt value is UTF-8.
//
// TODO: verify data is UTF-8.
while true {
if this.chunk_length <= 0 {
break.loop
} else if args.src.length() <= 0 {
yield? base."$short read"
continue.loop
} else if args.dst.length() <= 0 {
yield? base."$short write"
continue.loop
}
this.chunk_length -= 1
c = args.src.peek_u8()
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
args.dst.write_u8_fast!(a: c)
} endwhile
} else {
// Other uncompressed keys and values are Latin-1.
while true {
if this.chunk_length <= 0 {
// Keys are NUL-terminated but values are not.
@@ -1434,14 +1511,28 @@ pub func decoder.tell_me_more?(dst: base.io_writer, minfo: nptr base.more_inform
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
break.loop
}
// TODO: Convert from Latin-1 to UTF-8.
if args.dst.length() <= 0 {
yield? base."$short write"
continue.loop
c2 = LATIN_1[c]
if c2 == 0 {
return "#bad text chunk (not Latin-1)"
} else if c2 <= 0x7F {
if args.dst.length() <= 0 {
yield? base."$short write"
c2 = 0
continue.loop
}
this.chunk_length -= 1
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
args.dst.write_u8_fast!(a: c2 as base.u8)
} else {
if args.dst.length() <= 1 {
yield? base."$short write"
c2 = 0
continue.loop
}
this.chunk_length -= 1
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
args.dst.write_u16le_fast!(a: c2)
}
this.chunk_length -= 1
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
args.dst.write_u8_fast!(a: c)
} endwhile
}
} endwhile.loop