From 21bbbd422f70a4fc1f21b3c6b906b517592ce9b6 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 22 Oct 2017 14:44:53 +1100 Subject: [PATCH] Generate code for "mark in.dst". The generated code is ugly and the language design is not finalized, but the flate_backref_crosses_blocks test (testing a correctness bug) now passes. We can fix the ugliness in subsequent commits. --- cmd/puffs-c/internal/cgen/base-header.h | 10 ++++++ cmd/puffs-c/internal/cgen/data.go | 4 +-- cmd/puffs-c/internal/cgen/expr.go | 15 ++++++--- cmd/puffs-c/internal/cgen/statement.go | 3 +- cmd/puffs-c/internal/cgen/var.go | 16 +++++++-- gen/c/std/flate.c | 45 +++++++++++++++++++------ gen/c/std/gif.c | 10 ++++++ gen/h/std/flate.h | 10 ++++++ gen/h/std/gif.h | 10 ++++++ std/flate/decode_zlib.puffs | 3 ++ test/c/std/flate.c | 5 ++- 11 files changed, 107 insertions(+), 24 deletions(-) diff --git a/cmd/puffs-c/internal/cgen/base-header.h b/cmd/puffs-c/internal/cgen/base-header.h index a3b53d80..361f062f 100644 --- a/cmd/puffs-c/internal/cgen/base-header.h +++ b/cmd/puffs-c/internal/cgen/base-header.h @@ -79,11 +79,21 @@ typedef struct puffs_base__limit1 { typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__reader1; typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__writer1; #endif // PUFFS_BASE_HEADER_H diff --git a/cmd/puffs-c/internal/cgen/data.go b/cmd/puffs-c/internal/cgen/data.go index 4c35c143..b794fced 100644 --- a/cmd/puffs-c/internal/cgen/data.go +++ b/cmd/puffs-c/internal/cgen/data.go @@ -19,8 +19,8 @@ package cgen const baseHeader = "" + "#ifndef PUFFS_BASE_HEADER_H\n#define PUFFS_BASE_HEADER_H\n\n// Copyright 2017 The Puffs Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// https://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#include \n#include \n#include \n\n// Puffs requires a word size of at least 32 bits because it assumes that\n// converting a u32 to usize will never overflow. For example, the size of a\n// decoded image is often represented, explicitly or implicitly in an image\n// file, as a u32, and it is convenient to compare that to a buffer size.\n//\n// Si" + "milarly, the word size is at most 64 bits because it assumes that\n// converting a usize to u64 will never overflow.\n#if __WORDSIZE < 32\n#error \"Puffs requires a word size of at least 32 bits\"\n#elif __WORDSIZE > 64\n#error \"Puffs requires a word size of at most 64 bits\"\n#endif\n\n// PUFFS_VERSION is the major.minor version number as a uint32. The major\n// number is the high 16 bits. The minor number is the low 16 bits.\n//\n// The intention is to bump the version number at least on every API / ABI\n// backwards incompatible change.\n//\n// For now, the API and ABI are simply unstable and can change at any time.\n//\n// TODO: don't hard code this in base-header.h.\n#define PUFFS_VERSION (0x00001)\n\n// puffs_base__slice_u8 is a 1-dimensional buffer (a pointer and length).\n//\n// A value with all fields NULL or zero is a valid, empty slice.\ntypedef struct {\n uint8_t* ptr;\n size_t len;\n} puffs_base__slice_u8;\n\n// puffs_base__buf1 is a 1-dimensional buffer (a pointer and length), plus\n// additional indexes into that buffer, p" + - "lus an opened / closed flag.\n//\n// A value with all fields NULL or zero is a valid, empty buffer.\ntypedef struct {\n uint8_t* ptr; // Pointer.\n size_t len; // Length.\n size_t wi; // Write index. Invariant: wi <= len.\n size_t ri; // Read index. Invariant: ri <= wi.\n bool closed; // No further writes are expected.\n} puffs_base__buf1;\n\n// puffs_base__limit1 provides a limited view of a 1-dimensional byte stream:\n// its first N bytes. That N can be greater than a buffer's current read or\n// write capacity. N decreases naturally over time as bytes are read from or\n// written to the stream.\n//\n// A value with all fields NULL or zero is a valid, unlimited view.\ntypedef struct puffs_base__limit1 {\n uint64_t* ptr_to_len; // Pointer to N.\n struct puffs_base__limit1* next; // Linked list of limits.\n} puffs_base__limit1;\n\ntypedef struct {\n puffs_base__buf1* buf;\n puffs_base__limit1 limit;\n} puffs_base__reader1;\n\ntypedef struct {\n puffs_base__buf1* buf;\n puffs_base__limit1 limit;\n} pu" + - "ffs_base__writer1;\n\n#endif // PUFFS_BASE_HEADER_H\n" + + "lus an opened / closed flag.\n//\n// A value with all fields NULL or zero is a valid, empty buffer.\ntypedef struct {\n uint8_t* ptr; // Pointer.\n size_t len; // Length.\n size_t wi; // Write index. Invariant: wi <= len.\n size_t ri; // Read index. Invariant: ri <= wi.\n bool closed; // No further writes are expected.\n} puffs_base__buf1;\n\n// puffs_base__limit1 provides a limited view of a 1-dimensional byte stream:\n// its first N bytes. That N can be greater than a buffer's current read or\n// write capacity. N decreases naturally over time as bytes are read from or\n// written to the stream.\n//\n// A value with all fields NULL or zero is a valid, unlimited view.\ntypedef struct puffs_base__limit1 {\n uint64_t* ptr_to_len; // Pointer to N.\n struct puffs_base__limit1* next; // Linked list of limits.\n} puffs_base__limit1;\n\ntypedef struct {\n puffs_base__buf1* buf;\n puffs_base__limit1 limit;\n // Do not access the private_impl's fields directly. There is no API/ABI\n // compatibility or s" + + "afety guarantee if you do so.\n struct {\n uint8_t* mark;\n } private_impl;\n} puffs_base__reader1;\n\ntypedef struct {\n puffs_base__buf1* buf;\n puffs_base__limit1 limit;\n // Do not access the private_impl's fields directly. There is no API/ABI\n // compatibility or safety guarantee if you do so.\n struct {\n uint8_t* mark;\n } private_impl;\n} puffs_base__writer1;\n\n#endif // PUFFS_BASE_HEADER_H\n" + "" const baseImpl = "" + diff --git a/cmd/puffs-c/internal/cgen/expr.go b/cmd/puffs-c/internal/cgen/expr.go index 47b01eab..9f9c4a50 100644 --- a/cmd/puffs-c/internal/cgen/expr.go +++ b/cmd/puffs-c/internal/cgen/expr.go @@ -169,8 +169,10 @@ func (g *gen) writeExprOther(b *buffer, n *a.Expr, rp replacementPolicy, pp pare return nil } if isInDst(g.tm, n, t.KeySlice, 0) { - b.printf("((puffs_base__slice_u8){ .ptr = %swstart_dst, .len = %swptr_dst - %swstart_dst, })", - bPrefix, bPrefix, bPrefix) + b.printf("((puffs_base__slice_u8){ "+ + ".ptr = (%sdst.private_impl.mark ? %sdst.private_impl.mark : %swstart_dst), "+ + ".len = %swptr_dst - (%sdst.private_impl.mark ? %sdst.private_impl.mark : %swstart_dst), })", + aPrefix, aPrefix, bPrefix, bPrefix, aPrefix, aPrefix, bPrefix) return nil } if isInDst(g.tm, n, t.KeyCopyFromReader32, 2) { @@ -185,8 +187,9 @@ func (g *gen) writeExprOther(b *buffer, n *a.Expr, rp replacementPolicy, pp pare return nil } if isInDst(g.tm, n, t.KeyCopyFromHistory32, 2) { - b.printf("puffs_base__writer1__copy_from_history32(&%swptr_dst, %swstart_dst, %swend_dst", - bPrefix, bPrefix, bPrefix) + b.printf("puffs_base__writer1__copy_from_history32(&%swptr_dst, "+ + "(%sdst.private_impl.mark ? %sdst.private_impl.mark : %swstart_dst), %swend_dst", + bPrefix, aPrefix, aPrefix, bPrefix, bPrefix) for _, o := range n.Args() { b.writeb(',') if err := g.writeExpr(b, o.Arg().Value(), rp, parenthesesOptional, depth); err != nil { @@ -239,7 +242,9 @@ func (g *gen) writeExprOther(b *buffer, n *a.Expr, rp replacementPolicy, pp pare if typ := x.MType(); typ.Decorator() == 0 && typ.Name().Key() == t.KeyWriter1 { // TODO: don't hard-code dst. const wName = "dst" - b.printf("(uint64_t)(%swptr_%s - %swstart_%s)", bPrefix, wName, bPrefix, wName) + b.printf("(uint64_t)(%swptr_%s - "+ + "(%sdst.private_impl.mark ? %sdst.private_impl.mark : %swstart_%s))", + bPrefix, wName, aPrefix, aPrefix, bPrefix, wName) } else { b.writes("(uint64_t)(") if err := g.writeExpr(b, x, rp, parenthesesMandatory, depth); err != nil { diff --git a/cmd/puffs-c/internal/cgen/statement.go b/cmd/puffs-c/internal/cgen/statement.go index 232fa5c0..76bcb3da 100644 --- a/cmd/puffs-c/internal/cgen/statement.go +++ b/cmd/puffs-c/internal/cgen/statement.go @@ -69,7 +69,8 @@ func (g *gen) writeStatement(b *buffer, n *a.Node, depth uint32) error { return nil case a.KIO: - b.writes("/* placeholder */\n") + // TODO: the mark needs to apply on every (coroutine) call to the + // function, not just when we execute this particular statement. return nil case a.KIf: diff --git a/cmd/puffs-c/internal/cgen/var.go b/cmd/puffs-c/internal/cgen/var.go index 687d0d44..03a09689 100644 --- a/cmd/puffs-c/internal/cgen/var.go +++ b/cmd/puffs-c/internal/cgen/var.go @@ -88,7 +88,7 @@ func (g *gen) writeLoadDerivedVar(b *buffer, name t.ID, typ *a.TypeExpr, header case t.KeyReader1: if header { b.printf("uint8_t* %srptr_%s = NULL;", bPrefix, nameStr) - b.printf("uint8_t* %srstart_%s = NULL;", bPrefix, nameStr) + b.printf("uint8_t* %srstart_%s = NULL;", bPrefix, nameStr) // TODO: delete. b.printf("uint8_t* %srend_%s = NULL;", bPrefix, nameStr) } b.printf("if (%s%s.buf) {", aPrefix, nameStr) @@ -110,13 +110,25 @@ func (g *gen) writeLoadDerivedVar(b *buffer, name t.ID, typ *a.TypeExpr, header case t.KeyWriter1: if header { b.printf("uint8_t* %swptr_%s = NULL;", bPrefix, nameStr) - b.printf("uint8_t* %swstart_%s = NULL;", bPrefix, nameStr) + b.printf("uint8_t* %swstart_%s = NULL;", bPrefix, nameStr) // TODO: delete. b.printf("uint8_t* %swend_%s = NULL;", bPrefix, nameStr) } b.printf("if (%s%s.buf) {", aPrefix, nameStr) b.printf("%swptr_%s = %s%s.buf->ptr + %s%s.buf->wi;", bPrefix, nameStr, aPrefix, nameStr, aPrefix, nameStr) + + // TODO: remove this hard-coded hack for "mark in.dst". + // + // TODO: is a private_impl.mark the right representation? What if the + // function is passed a (ptr writer1) instead of a (writer1)? Do we + // still want to have that mark live outside of the function scope? + if header && + g.currFunk.astFunc.Receiver().String(g.tm) == "flate_decoder" && + g.currFunk.astFunc.Name().String(g.tm) == "decode" { + b.printf("%s%s.private_impl.mark = %swptr_%s;", aPrefix, nameStr, bPrefix, nameStr) + } + if header { b.printf("%swstart_%s = %swptr_%s;", bPrefix, nameStr, bPrefix, nameStr) } diff --git a/gen/c/std/flate.c b/gen/c/std/flate.c index 457ccc41..06fd813f 100644 --- a/gen/c/std/flate.c +++ b/gen/c/std/flate.c @@ -82,11 +82,21 @@ typedef struct puffs_base__limit1 { typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__reader1; typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__writer1; #endif // PUFFS_BASE_HEADER_H @@ -807,6 +817,7 @@ puffs_flate__status puffs_flate__flate_decoder__decode( uint8_t* b_wend_dst = NULL; if (a_dst.buf) { b_wptr_dst = a_dst.buf->ptr + a_dst.buf->wi; + a_dst.private_impl.mark = b_wptr_dst; b_wstart_dst = b_wptr_dst; b_wend_dst = b_wptr_dst; if (!a_dst.buf->closed) { @@ -831,7 +842,6 @@ puffs_flate__status puffs_flate__flate_decoder__decode( switch (coro_susp_point) { PUFFS_BASE__COROUTINE_SUSPENSION_POINT(0); - /* placeholder */ while (true) { PUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); if (a_dst.buf) { @@ -866,8 +876,10 @@ puffs_flate__status puffs_flate__flate_decoder__decode( PUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2); } v_written = ((puffs_base__slice_u8){ - .ptr = b_wstart_dst, - .len = b_wptr_dst - b_wstart_dst, + .ptr = (a_dst.private_impl.mark ? a_dst.private_impl.mark + : b_wstart_dst), + .len = b_wptr_dst - (a_dst.private_impl.mark ? a_dst.private_impl.mark + : b_wstart_dst), }); if (((uint64_t)(v_written.len)) >= 32768) { v_written = puffs_base__slice_u8_suffix(v_written, 32768); @@ -1539,10 +1551,15 @@ puffs_flate__status puffs_flate__flate_decoder__decode_huffman( v_n_copied = 0; while (true) { if (((uint64_t)(v_distance)) > - ((uint64_t)(b_wptr_dst - b_wstart_dst))) { + ((uint64_t)(b_wptr_dst - (a_dst.private_impl.mark + ? a_dst.private_impl.mark + : b_wstart_dst)))) { v_hlen = 0; - v_hdist = ((uint32_t)((((uint64_t)(v_distance)) - - ((uint64_t)(b_wptr_dst - b_wstart_dst))))); + v_hdist = ((uint32_t)( + (((uint64_t)(v_distance)) - + ((uint64_t)(b_wptr_dst - (a_dst.private_impl.mark + ? a_dst.private_impl.mark + : b_wstart_dst)))))); if (v_length > v_hdist) { v_length -= v_hdist; v_hlen = v_hdist; @@ -1603,7 +1620,9 @@ puffs_flate__status puffs_flate__flate_decoder__decode_huffman( } } v_n_copied = puffs_base__writer1__copy_from_history32( - &b_wptr_dst, b_wstart_dst, b_wend_dst, v_distance, v_length); + &b_wptr_dst, + (a_dst.private_impl.mark ? a_dst.private_impl.mark : b_wstart_dst), + b_wend_dst, v_distance, v_length); if (v_length <= v_n_copied) { v_length = 0; goto label_7_break; @@ -2433,10 +2452,14 @@ puffs_flate__status puffs_flate__zlib_decoder__decode( } v_z = t_2; v_checksum = puffs_flate__adler32__update( - &self->private_impl.f_adler, ((puffs_base__slice_u8){ - .ptr = b_wstart_dst, - .len = b_wptr_dst - b_wstart_dst, - })); + &self->private_impl.f_adler, + ((puffs_base__slice_u8){ + .ptr = (a_dst.private_impl.mark ? a_dst.private_impl.mark + : b_wstart_dst), + .len = b_wptr_dst - (a_dst.private_impl.mark + ? a_dst.private_impl.mark + : b_wstart_dst), + })); if (v_z == 0) { goto label_0_break; } diff --git a/gen/c/std/gif.c b/gen/c/std/gif.c index 78151640..8339241f 100644 --- a/gen/c/std/gif.c +++ b/gen/c/std/gif.c @@ -82,11 +82,21 @@ typedef struct puffs_base__limit1 { typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__reader1; typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__writer1; #endif // PUFFS_BASE_HEADER_H diff --git a/gen/h/std/flate.h b/gen/h/std/flate.h index b9feb55d..878ac234 100644 --- a/gen/h/std/flate.h +++ b/gen/h/std/flate.h @@ -82,11 +82,21 @@ typedef struct puffs_base__limit1 { typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__reader1; typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__writer1; #endif // PUFFS_BASE_HEADER_H diff --git a/gen/h/std/gif.h b/gen/h/std/gif.h index 91d75b73..3daa02bd 100644 --- a/gen/h/std/gif.h +++ b/gen/h/std/gif.h @@ -82,11 +82,21 @@ typedef struct puffs_base__limit1 { typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__reader1; typedef struct { puffs_base__buf1* buf; puffs_base__limit1 limit; + // Do not access the private_impl's fields directly. There is no API/ABI + // compatibility or safety guarantee if you do so. + struct { + uint8_t* mark; + } private_impl; } puffs_base__writer1; #endif // PUFFS_BASE_HEADER_H diff --git a/std/flate/decode_zlib.puffs b/std/flate/decode_zlib.puffs index 4d63bbae..5788ff7e 100644 --- a/std/flate/decode_zlib.puffs +++ b/std/flate/decode_zlib.puffs @@ -49,6 +49,9 @@ pub func zlib_decoder.decode?(dst writer1, src reader1)() { var z status = try this.flate.decode?(dst:in.dst, src:in.src) // TODO: expose some API to opt out of checksumming, trading off error // detection for throughput. + // + // TODO: in.dst.slice needs a previous mark statement. Do that, and + // remove the b_wstart_dst derived variables. checksum = this.adler.update(x:in.dst.slice()) if z.is_ok() { break diff --git a/test/c/std/flate.c b/test/c/std/flate.c index 6a02a19f..655f135c 100644 --- a/test/c/std/flate.c +++ b/test/c/std/flate.c @@ -255,9 +255,8 @@ void test_puffs_flate_decode_256_bytes() { void test_puffs_flate_decode_flate_backref_crosses_blocks() { proc_funcname = __func__; - // TODO: uncomment. - // do_test_buf1_buf1(puffs_flate_decode, - // &flate_flate_backref_crosses_blocks_gt, 0, 0); + do_test_buf1_buf1(puffs_flate_decode, &flate_flate_backref_crosses_blocks_gt, + 0, 0); } void test_puffs_flate_decode_midsummer() {