From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 77EFC3D6485; Tue, 2 Jun 2026 17:30:16 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1780421477; cv=none; b=SdMs5fSezztnUoNOZ2EbafyVRzj63utIaE79cBj9TI0ZWe2IgMIGG+91INDOtP+asp1ys5eE3aEl6F3arpPmGtEDleMd1Ei7mdUcX154fWdXcKPog7X4jyrWK+xyRbO8L71gThdd0tqH+FCQXNkvJ5wzBlIdOaZwsFYA1L1iFpY= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1780421477; c=relaxed/simple; bh=mq/VUbOqaJ7bsDHak5T3ctBV+9aLhjys0Uri5bEZczg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=boROTU/ZF3rzxJ888yrnI66O0nQXH42/xmMZSsMMhsIv9m/y8XrXlCT/8XmubS286/IWvAkcLl+81RkvSZ64xPSAzEe4rqsVcNbFB7gwksfBajLbVandPej9AzC/vu3gpXbiCZOzoKykGrUC6dertM6y8+y2Q8XIW9BrroDLXz4= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=LRPaWQ+9; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="LRPaWQ+9" Received: by smtp.kernel.org (Postfix) with ESMTPSA id F03051F00893; Tue, 2 Jun 2026 17:30:11 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1780421416; bh=nONRAFueLz76Q+D9Ld1VnMz2m69HcIhRwZQMvu3UCmE=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=LRPaWQ+9EejM3H7Tcot4caMgnOA6gUa7XEQDA/nuX1q988W/KdpH6PbdvBwJrKIZy 3rdjNuiZk3j4MGJALQTgRSDGPwFT0hD1XXdvSuyWOtdcbR9YC07TDpLJRLBj5r+UnW irD8AsTlLRCgC6tWZyFWcaay9rnbMtT65aArm8I+yRtJ3AX1UxzYi6fu3bOeaJyMJW ZFj5UmTu+vTd7ixrsgPvuc2f7f2ockk2bcLNev43+0d4J+L9d/hmyf90udni6fJPt/ Py6BM+VW8iJwB3Uax1PQQKo5GJ340svxlj9z7EdhLVb0fqA+P92JzORRn3Oxgh5w1R 1IvlUFX8CFJWw== From: Miguel Ojeda To: Miguel Ojeda , Nathan Chancellor , Nicolas Schier Cc: Boqun Feng , Gary Guo , =?UTF-8?q?Bj=C3=B6rn=20Roy=20Baron?= , Benno Lossin , Andreas Hindborg , Alice Ryhl , Trevor Gross , Danilo Krummrich , rust-for-linux@vger.kernel.org, linux-kbuild@vger.kernel.org, Joshua Liebow-Feeser , Jack Wrenn Subject: [PATCH 08/18] rust: zerocopy: import crate Date: Tue, 2 Jun 2026 19:29:09 +0200 Message-ID: <20260602172920.30342-9-ojeda@kernel.org> In-Reply-To: <20260602172920.30342-1-ojeda@kernel.org> References: <20260602172920.30342-1-ojeda@kernel.org> Precedence: bulk X-Mailing-List: linux-kbuild@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a subset of the Rust `zerocopy` crate, version v0.8.50 (released 2026-05-31), licensed under "BSD-2-Clause OR Apache-2.0 OR MIT", from: https://github.com/google/zerocopy/tree/v0.8.50 The files are copied as-is, with no modifications whatsoever (not even adding the SPDX identifiers). The `benches` folder is added (i.e. not just `src` like in other cases) since the files there are included in the rendered documentation, as well as the `rustdoc` CSS style file that is needed to make those visually more understandable. For copyright details, please see: https://github.com/google/zerocopy/blob/v0.8.50/README.md?plain=1 https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-BSD https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-APACHE https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-MIT The next two patches modify these files as needed for use within the kernel. This patch split allows reviewers to double-check the import and to clearly see the differences introduced. The following script may be used to verify the contents: for path in $(cd rust/zerocopy/ && find . -type f); do curl --silent --show-error --location \ https://github.com/google/zerocopy/raw/v0.8.50/$path \ | diff --unified rust/zerocopy/$path - && echo $path: OK done Cc: Joshua Liebow-Feeser Cc: Jack Wrenn Signed-off-by: Miguel Ojeda --- .../zerocopy/benches/as_bytes_dynamic_size.rs | 9 + .../benches/as_bytes_dynamic_size.x86-64 | 5 + .../benches/as_bytes_dynamic_size.x86-64.mca | 47 + rust/zerocopy/benches/as_bytes_static_size.rs | 9 + .../benches/as_bytes_static_size.x86-64 | 4 + .../benches/as_bytes_static_size.x86-64.mca | 45 + rust/zerocopy/benches/extend_vec_zeroed.rs | 9 + .../zerocopy/benches/extend_vec_zeroed.x86-64 | 60 + .../benches/extend_vec_zeroed.x86-64.mca | 147 + .../benches/formats/coco_dynamic_padding.rs | 24 + .../benches/formats/coco_dynamic_size.rs | 27 + .../benches/formats/coco_static_size.rs | 27 + rust/zerocopy/benches/insert_vec_zeroed.rs | 13 + .../zerocopy/benches/insert_vec_zeroed.x86-64 | 79 + .../benches/insert_vec_zeroed.x86-64.mca | 183 + rust/zerocopy/benches/new_box_zeroed.rs | 9 + rust/zerocopy/benches/new_box_zeroed.x86-64 | 7 + .../benches/new_box_zeroed.x86-64.mca | 51 + ...w_box_zeroed_with_elems_dynamic_padding.rs | 11 + ...x_zeroed_with_elems_dynamic_padding.x86-64 | 24 + ...roed_with_elems_dynamic_padding.x86-64.mca | 81 + .../new_box_zeroed_with_elems_dynamic_size.rs | 9 + ..._box_zeroed_with_elems_dynamic_size.x86-64 | 22 + ..._zeroed_with_elems_dynamic_size.x86-64.mca | 77 + rust/zerocopy/benches/new_vec_zeroed.rs | 9 + rust/zerocopy/benches/new_vec_zeroed.x86-64 | 40 + .../benches/new_vec_zeroed.x86-64.mca | 113 + rust/zerocopy/benches/new_zeroed.rs | 9 + rust/zerocopy/benches/new_zeroed.x86-64 | 3 + rust/zerocopy/benches/new_zeroed.x86-64.mca | 43 + rust/zerocopy/benches/read_from_bytes.rs | 7 + rust/zerocopy/benches/read_from_bytes.x86-64 | 15 + .../benches/read_from_bytes.x86-64.mca | 65 + rust/zerocopy/benches/read_from_prefix.rs | 10 + rust/zerocopy/benches/read_from_prefix.x86-64 | 14 + .../benches/read_from_prefix.x86-64.mca | 63 + rust/zerocopy/benches/read_from_suffix.rs | 10 + rust/zerocopy/benches/read_from_suffix.x86-64 | 15 + .../benches/read_from_suffix.x86-64.mca | 65 + .../benches/ref_from_bytes_dynamic_padding.rs | 7 + .../ref_from_bytes_dynamic_padding.x86-64 | 22 + .../ref_from_bytes_dynamic_padding.x86-64.mca | 77 + .../benches/ref_from_bytes_dynamic_size.rs | 7 + .../ref_from_bytes_dynamic_size.x86-64 | 20 + .../ref_from_bytes_dynamic_size.x86-64.mca | 75 + .../benches/ref_from_bytes_static_size.rs | 7 + .../benches/ref_from_bytes_static_size.x86-64 | 8 + .../ref_from_bytes_static_size.x86-64.mca | 53 + ...f_from_bytes_with_elems_dynamic_padding.rs | 10 + ...om_bytes_with_elems_dynamic_padding.x86-64 | 19 + ...ytes_with_elems_dynamic_padding.x86-64.mca | 71 + .../ref_from_bytes_with_elems_dynamic_size.rs | 10 + ..._from_bytes_with_elems_dynamic_size.x86-64 | 16 + ...m_bytes_with_elems_dynamic_size.x86-64.mca | 65 + .../ref_from_prefix_dynamic_padding.rs | 10 + .../ref_from_prefix_dynamic_padding.x86-64 | 22 + ...ref_from_prefix_dynamic_padding.x86-64.mca | 77 + .../benches/ref_from_prefix_dynamic_size.rs | 10 + .../ref_from_prefix_dynamic_size.x86-64 | 17 + .../ref_from_prefix_dynamic_size.x86-64.mca | 67 + .../benches/ref_from_prefix_static_size.rs | 10 + .../ref_from_prefix_static_size.x86-64 | 8 + .../ref_from_prefix_static_size.x86-64.mca | 53 + ..._from_prefix_with_elems_dynamic_padding.rs | 13 + ...m_prefix_with_elems_dynamic_padding.x86-64 | 26 + ...efix_with_elems_dynamic_padding.x86-64.mca | 85 + ...ref_from_prefix_with_elems_dynamic_size.rs | 13 + ...from_prefix_with_elems_dynamic_size.x86-64 | 22 + ..._prefix_with_elems_dynamic_size.x86-64.mca | 77 + .../ref_from_suffix_dynamic_padding.rs | 10 + .../ref_from_suffix_dynamic_padding.x86-64 | 23 + ...ref_from_suffix_dynamic_padding.x86-64.mca | 79 + .../benches/ref_from_suffix_dynamic_size.rs | 10 + .../ref_from_suffix_dynamic_size.x86-64 | 13 + .../ref_from_suffix_dynamic_size.x86-64.mca | 63 + .../benches/ref_from_suffix_static_size.rs | 10 + .../ref_from_suffix_static_size.x86-64 | 13 + .../ref_from_suffix_static_size.x86-64.mca | 61 + ..._from_suffix_with_elems_dynamic_padding.rs | 13 + ...m_suffix_with_elems_dynamic_padding.x86-64 | 27 + ...ffix_with_elems_dynamic_padding.x86-64.mca | 85 + ...ref_from_suffix_with_elems_dynamic_size.rs | 13 + ...from_suffix_with_elems_dynamic_size.x86-64 | 23 + ..._suffix_with_elems_dynamic_size.x86-64.mca | 77 + .../benches/split_at_dynamic_padding.rs | 12 + .../benches/split_at_dynamic_padding.x86-64 | 12 + .../split_at_dynamic_padding.x86-64.mca | 59 + .../zerocopy/benches/split_at_dynamic_size.rs | 12 + .../benches/split_at_dynamic_size.x86-64 | 12 + .../benches/split_at_dynamic_size.x86-64.mca | 59 + .../split_at_unchecked_dynamic_padding.rs | 12 + .../split_at_unchecked_dynamic_padding.x86-64 | 6 + ...it_at_unchecked_dynamic_padding.x86-64.mca | 49 + .../split_at_unchecked_dynamic_size.rs | 12 + .../split_at_unchecked_dynamic_size.x86-64 | 6 + ...split_at_unchecked_dynamic_size.x86-64.mca | 49 + .../split_via_immutable_dynamic_padding.rs | 11 + ...split_via_immutable_dynamic_padding.x86-64 | 14 + ...t_via_immutable_dynamic_padding.x86-64.mca | 65 + .../split_via_immutable_dynamic_size.rs | 11 + .../split_via_immutable_dynamic_size.x86-64 | 13 + ...plit_via_immutable_dynamic_size.x86-64.mca | 63 + ...split_via_runtime_check_dynamic_padding.rs | 11 + ...t_via_runtime_check_dynamic_padding.x86-64 | 22 + ...a_runtime_check_dynamic_padding.x86-64.mca | 79 + .../split_via_runtime_check_dynamic_size.rs | 11 + ...plit_via_runtime_check_dynamic_size.x86-64 | 13 + ..._via_runtime_check_dynamic_size.x86-64.mca | 63 + .../split_via_unchecked_dynamic_padding.rs | 11 + ...split_via_unchecked_dynamic_padding.x86-64 | 14 + ...t_via_unchecked_dynamic_padding.x86-64.mca | 65 + .../split_via_unchecked_dynamic_size.rs | 11 + .../split_via_unchecked_dynamic_size.x86-64 | 13 + ...plit_via_unchecked_dynamic_size.x86-64.mca | 63 + rust/zerocopy/benches/transmute.rs | 16 + rust/zerocopy/benches/transmute.x86-64 | 3 + rust/zerocopy/benches/transmute.x86-64.mca | 43 + .../benches/transmute_ref_dynamic_size.rs | 16 + .../benches/transmute_ref_dynamic_size.x86-64 | 4 + .../transmute_ref_dynamic_size.x86-64.mca | 45 + .../benches/transmute_ref_static_size.rs | 15 + .../benches/transmute_ref_static_size.x86-64 | 3 + .../transmute_ref_static_size.x86-64.mca | 43 + rust/zerocopy/benches/try_read_from_bytes.rs | 7 + .../benches/try_read_from_bytes.x86-64 | 23 + .../benches/try_read_from_bytes.x86-64.mca | 79 + rust/zerocopy/benches/try_read_from_prefix.rs | 10 + .../benches/try_read_from_prefix.x86-64 | 16 + .../benches/try_read_from_prefix.x86-64.mca | 67 + rust/zerocopy/benches/try_read_from_suffix.rs | 10 + .../benches/try_read_from_suffix.x86-64 | 18 + .../benches/try_read_from_suffix.x86-64.mca | 71 + .../try_ref_from_bytes_dynamic_padding.rs | 7 + .../try_ref_from_bytes_dynamic_padding.x86-64 | 24 + ..._ref_from_bytes_dynamic_padding.x86-64.mca | 81 + .../try_ref_from_bytes_dynamic_size.rs | 7 + .../try_ref_from_bytes_dynamic_size.x86-64 | 22 + ...try_ref_from_bytes_dynamic_size.x86-64.mca | 79 + .../benches/try_ref_from_bytes_static_size.rs | 7 + .../try_ref_from_bytes_static_size.x86-64 | 13 + .../try_ref_from_bytes_static_size.x86-64.mca | 59 + ...f_from_bytes_with_elems_dynamic_padding.rs | 10 + ...om_bytes_with_elems_dynamic_padding.x86-64 | 21 + ...ytes_with_elems_dynamic_padding.x86-64.mca | 75 + ..._ref_from_bytes_with_elems_dynamic_size.rs | 10 + ..._from_bytes_with_elems_dynamic_size.x86-64 | 18 + ...m_bytes_with_elems_dynamic_size.x86-64.mca | 69 + .../try_ref_from_prefix_dynamic_padding.rs | 10 + ...try_ref_from_prefix_dynamic_padding.x86-64 | 29 + ...ref_from_prefix_dynamic_padding.x86-64.mca | 91 + .../try_ref_from_prefix_dynamic_size.rs | 10 + .../try_ref_from_prefix_dynamic_size.x86-64 | 22 + ...ry_ref_from_prefix_dynamic_size.x86-64.mca | 77 + .../try_ref_from_prefix_static_size.rs | 10 + .../try_ref_from_prefix_static_size.x86-64 | 15 + ...try_ref_from_prefix_static_size.x86-64.mca | 63 + ..._from_prefix_with_elems_dynamic_padding.rs | 13 + ...m_prefix_with_elems_dynamic_padding.x86-64 | 30 + ...efix_with_elems_dynamic_padding.x86-64.mca | 91 + ...ref_from_prefix_with_elems_dynamic_size.rs | 13 + ...from_prefix_with_elems_dynamic_size.x86-64 | 26 + ..._prefix_with_elems_dynamic_size.x86-64.mca | 83 + .../try_ref_from_suffix_dynamic_padding.rs | 10 + ...try_ref_from_suffix_dynamic_padding.x86-64 | 26 + ...ref_from_suffix_dynamic_padding.x86-64.mca | 85 + .../try_ref_from_suffix_dynamic_size.rs | 10 + .../try_ref_from_suffix_dynamic_size.x86-64 | 18 + ...ry_ref_from_suffix_dynamic_size.x86-64.mca | 71 + .../try_ref_from_suffix_static_size.rs | 10 + .../try_ref_from_suffix_static_size.x86-64 | 16 + ...try_ref_from_suffix_static_size.x86-64.mca | 67 + ..._from_suffix_with_elems_dynamic_padding.rs | 13 + ...m_suffix_with_elems_dynamic_padding.x86-64 | 32 + ...ffix_with_elems_dynamic_padding.x86-64.mca | 95 + ...ref_from_suffix_with_elems_dynamic_size.rs | 13 + ...from_suffix_with_elems_dynamic_size.x86-64 | 28 + ..._suffix_with_elems_dynamic_size.x86-64.mca | 87 + rust/zerocopy/benches/try_transmute.rs | 16 + rust/zerocopy/benches/try_transmute.x86-64 | 9 + .../zerocopy/benches/try_transmute.x86-64.mca | 55 + .../benches/try_transmute_ref_dynamic_size.rs | 18 + .../try_transmute_ref_dynamic_size.x86-64 | 6 + .../try_transmute_ref_dynamic_size.x86-64.mca | 49 + .../benches/try_transmute_ref_static_size.rs | 17 + .../try_transmute_ref_static_size.x86-64 | 5 + .../try_transmute_ref_static_size.x86-64.mca | 47 + .../zerocopy/benches/write_to_dynamic_size.rs | 9 + .../benches/write_to_dynamic_size.x86-64 | 21 + .../benches/write_to_dynamic_size.x86-64.mca | 77 + .../benches/write_to_prefix_dynamic_size.rs | 12 + .../write_to_prefix_dynamic_size.x86-64 | 21 + .../write_to_prefix_dynamic_size.x86-64.mca | 77 + .../benches/write_to_prefix_static_size.rs | 12 + .../write_to_prefix_static_size.x86-64 | 11 + .../write_to_prefix_static_size.x86-64.mca | 57 + rust/zerocopy/benches/write_to_static_size.rs | 9 + .../benches/write_to_static_size.x86-64 | 11 + .../benches/write_to_static_size.x86-64.mca | 57 + .../benches/write_to_suffix_dynamic_size.rs | 12 + .../write_to_suffix_dynamic_size.x86-64 | 22 + .../write_to_suffix_dynamic_size.x86-64.mca | 79 + .../benches/write_to_suffix_static_size.rs | 12 + .../write_to_suffix_static_size.x86-64 | 11 + .../write_to_suffix_static_size.x86-64.mca | 57 + rust/zerocopy/benches/zero_dynamic_padding.rs | 9 + .../benches/zero_dynamic_padding.x86-64 | 7 + .../benches/zero_dynamic_padding.x86-64.mca | 51 + rust/zerocopy/benches/zero_dynamic_size.rs | 9 + .../zerocopy/benches/zero_dynamic_size.x86-64 | 5 + .../benches/zero_dynamic_size.x86-64.mca | 47 + rust/zerocopy/benches/zero_static_size.rs | 9 + rust/zerocopy/benches/zero_static_size.x86-64 | 4 + .../benches/zero_static_size.x86-64.mca | 45 + rust/zerocopy/rustdoc/style.css | 55 + rust/zerocopy/src/byte_slice.rs | 432 + rust/zerocopy/src/byteorder.rs | 1563 ++++ rust/zerocopy/src/deprecated.rs | 279 + rust/zerocopy/src/error.rs | 1348 +++ rust/zerocopy/src/impls.rs | 2387 ++++++ rust/zerocopy/src/layout.rs | 2223 +++++ rust/zerocopy/src/lib.rs | 7610 +++++++++++++++++ rust/zerocopy/src/macros.rs | 1823 ++++ rust/zerocopy/src/pointer/inner.rs | 752 ++ rust/zerocopy/src/pointer/invariant.rs | 296 + rust/zerocopy/src/pointer/mod.rs | 408 + rust/zerocopy/src/pointer/ptr.rs | 1584 ++++ rust/zerocopy/src/pointer/transmute.rs | 520 ++ rust/zerocopy/src/ref.rs | 1356 +++ rust/zerocopy/src/split_at.rs | 1088 +++ rust/zerocopy/src/util/macro_util.rs | 1308 +++ rust/zerocopy/src/util/macros.rs | 1065 +++ rust/zerocopy/src/util/mod.rs | 936 ++ rust/zerocopy/src/wrappers.rs | 1032 +++ 233 files changed, 35005 insertions(+) create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/as_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/as_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/as_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/formats/coco_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/formats/coco_dynamic_size.rs create mode 100644 rust/zerocopy/benches/formats/coco_static_size.rs create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/new_vec_zeroed.rs create mode 100644 rust/zerocopy/benches/new_vec_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_vec_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/new_zeroed.rs create mode 100644 rust/zerocopy/benches/new_zeroed.x86-64 create mode 100644 rust/zerocopy/benches/new_zeroed.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_bytes.rs create mode 100644 rust/zerocopy/benches/read_from_bytes.x86-64 create mode 100644 rust/zerocopy/benches/read_from_bytes.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_prefix.rs create mode 100644 rust/zerocopy/benches/read_from_prefix.x86-64 create mode 100644 rust/zerocopy/benches/read_from_prefix.x86-64.mca create mode 100644 rust/zerocopy/benches/read_from_suffix.rs create mode 100644 rust/zerocopy/benches/read_from_suffix.x86-64 create mode 100644 rust/zerocopy/benches/read_from_suffix.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute.rs create mode 100644 rust/zerocopy/benches/transmute.x86-64 create mode 100644 rust/zerocopy/benches/transmute.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.rs create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.rs create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.x86-64 create mode 100644 rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_bytes.rs create mode 100644 rust/zerocopy/benches/try_read_from_bytes.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_bytes.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_prefix.rs create mode 100644 rust/zerocopy/benches/try_read_from_prefix.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_prefix.x86-64.mca create mode 100644 rust/zerocopy/benches/try_read_from_suffix.rs create mode 100644 rust/zerocopy/benches/try_read_from_suffix.x86-64 create mode 100644 rust/zerocopy/benches/try_read_from_suffix.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute.rs create mode 100644 rust/zerocopy/benches/try_transmute.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.rs create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 create mode 100644 rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.rs create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.rs create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.x86-64 create mode 100644 rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.rs create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.x86-64 create mode 100644 rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_dynamic_size.rs create mode 100644 rust/zerocopy/benches/zero_dynamic_size.x86-64 create mode 100644 rust/zerocopy/benches/zero_dynamic_size.x86-64.mca create mode 100644 rust/zerocopy/benches/zero_static_size.rs create mode 100644 rust/zerocopy/benches/zero_static_size.x86-64 create mode 100644 rust/zerocopy/benches/zero_static_size.x86-64.mca create mode 100644 rust/zerocopy/rustdoc/style.css create mode 100644 rust/zerocopy/src/byte_slice.rs create mode 100644 rust/zerocopy/src/byteorder.rs create mode 100644 rust/zerocopy/src/deprecated.rs create mode 100644 rust/zerocopy/src/error.rs create mode 100644 rust/zerocopy/src/impls.rs create mode 100644 rust/zerocopy/src/layout.rs create mode 100644 rust/zerocopy/src/lib.rs create mode 100644 rust/zerocopy/src/macros.rs create mode 100644 rust/zerocopy/src/pointer/inner.rs create mode 100644 rust/zerocopy/src/pointer/invariant.rs create mode 100644 rust/zerocopy/src/pointer/mod.rs create mode 100644 rust/zerocopy/src/pointer/ptr.rs create mode 100644 rust/zerocopy/src/pointer/transmute.rs create mode 100644 rust/zerocopy/src/ref.rs create mode 100644 rust/zerocopy/src/split_at.rs create mode 100644 rust/zerocopy/src/util/macro_util.rs create mode 100644 rust/zerocopy/src/util/macros.rs create mode 100644 rust/zerocopy/src/util/mod.rs create mode 100644 rust/zerocopy/src/wrappers.rs diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.rs b/rust/zerocopy/benches/as_bytes_dynamic_size.rs new file mode 100644 index 000000000000..68cd1d6f4111 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_as_bytes_dynamic_size(source: &format::CocoPacket) -> &[u8] { + source.as_bytes() +} diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..f68bad612695 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 @@ -0,0 +1,5 @@ +bench_as_bytes_dynamic_size: + mov rax, rdi + lea rdx, [2*rsi + 5] + and rdx, -2 + ret diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..c3b92a9a95fe --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 137 +Total uOps: 400 + +Dispatch Width: 4 +uOps Per Cycle: 2.92 +IPC: 2.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdx, [2*rsi + 5] + 1 1 0.33 and rdx, -2 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.33 1.33 - 1.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.66 - 0.34 - - mov rax, rdi + - - 0.33 0.67 - - - - lea rdx, [2*rsi + 5] + - - 1.00 - - - - - and rdx, -2 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/as_bytes_static_size.rs b/rust/zerocopy/benches/as_bytes_static_size.rs new file mode 100644 index 000000000000..2ad738e95480 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_as_bytes_static_size(source: &format::CocoPacket) -> &[u8] { + source.as_bytes() +} diff --git a/rust/zerocopy/benches/as_bytes_static_size.x86-64 b/rust/zerocopy/benches/as_bytes_static_size.x86-64 new file mode 100644 index 000000000000..213e74ab54ff --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.x86-64 @@ -0,0 +1,4 @@ +bench_as_bytes_static_size: + mov rax, rdi + mov edx, 6 + ret diff --git a/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..ae04a6ba9061 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 104 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.88 +IPC: 2.88 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov edx, 6 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - mov edx, 6 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/extend_vec_zeroed.rs b/rust/zerocopy/benches/extend_vec_zeroed.rs new file mode 100644 index 000000000000..1fbf772d1ea7 --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_extend_vec_zeroed(v: &mut Vec, additional: usize) -> Option<()> { + FromZeros::extend_vec_zeroed(v, additional).ok() +} diff --git a/rust/zerocopy/benches/extend_vec_zeroed.x86-64 b/rust/zerocopy/benches/extend_vec_zeroed.x86-64 new file mode 100644 index 000000000000..831b2a075fec --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.x86-64 @@ -0,0 +1,60 @@ +bench_extend_vec_zeroed: + push r15 + push r14 + push r13 + push r12 + push rbx + sub rsp, 32 + mov rbx, rdi + mov rax, qword ptr [rdi] + mov r12, qword ptr [rdi + 16] + mov rcx, rax + sub rcx, r12 + cmp rsi, rcx + jbe .LBB6_3 + mov r15, r12 + add r15, rsi + jae .LBB6_6 +.LBB6_2: + xor eax, eax + jmp .LBB6_5 +.LBB6_3: + mov rax, qword ptr [rbx + 8] + lea r15, [r12 + rsi] +.LBB6_4: + lea rcx, [r12 + 2*r12] + lea rdi, [rax + 2*rcx] + add rsi, rsi + lea rdx, [rsi + 2*rsi] + xor esi, esi + call qword ptr [rip + memset@GOTPCREL] + mov qword ptr [rbx + 16], r15 + mov al, 1 +.LBB6_5: + add rsp, 32 + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.LBB6_6: + mov r13, rsi + lea rcx, [rax + rax] + cmp r15, rcx + cmova rcx, r15 + cmp rcx, 5 + mov r14d, 4 + cmovae r14, rcx + mov rdx, qword ptr [rbx + 8] + lea rdi, [rsp + 8] + mov rsi, rax + mov rcx, r14 + call ::finish_grow + cmp dword ptr [rsp + 8], 1 + je .LBB6_2 + mov rax, qword ptr [rsp + 16] + mov qword ptr [rbx + 8], rax + mov qword ptr [rbx], r14 + mov rsi, r13 + jmp .LBB6_4 diff --git a/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..cfab1eea8f56 --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca @@ -0,0 +1,147 @@ +Iterations: 100 +Instructions: 5400 +Total Cycles: 6595 +Total uOps: 6800 + +Dispatch Width: 4 +uOps Per Cycle: 1.03 +IPC: 0.82 +Block RThroughput: 17.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push r13 + 2 5 1.00 * push r12 + 2 5 1.00 * push rbx + 1 1 0.33 sub rsp, 32 + 1 1 0.33 mov rbx, rdi + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov r12, qword ptr [rdi + 16] + 1 1 0.33 mov rcx, rax + 1 1 0.33 sub rcx, r12 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jbe .LBB6_3 + 1 1 0.33 mov r15, r12 + 1 1 0.33 add r15, rsi + 1 1 1.00 jae .LBB6_6 + 1 0 0.25 xor eax, eax + 1 1 1.00 jmp .LBB6_5 + 1 5 0.50 * mov rax, qword ptr [rbx + 8] + 1 1 0.50 lea r15, [r12 + rsi] + 1 1 0.50 lea rcx, [r12 + 2*r12] + 1 1 0.50 lea rdi, [rax + 2*rcx] + 1 1 0.33 add rsi, rsi + 1 1 0.50 lea rdx, [rsi + 2*rsi] + 1 0 0.25 xor esi, esi + 4 7 1.00 * call qword ptr [rip + memset@GOTPCREL] + 1 1 1.00 * mov qword ptr [rbx + 16], r15 + 1 1 0.33 mov al, 1 + 1 1 0.33 add rsp, 32 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r12 + 1 6 0.50 * pop r13 + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 1 1.00 U ret + 1 1 0.33 mov r13, rsi + 1 1 0.50 lea rcx, [rax + rax] + 1 1 0.33 cmp r15, rcx + 3 3 1.00 cmova rcx, r15 + 1 1 0.33 cmp rcx, 5 + 1 1 0.33 mov r14d, 4 + 2 2 0.67 cmovae r14, rcx + 1 5 0.50 * mov rdx, qword ptr [rbx + 8] + 1 1 0.50 lea rdi, [rsp + 8] + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rcx, r14 + 3 5 1.00 call ::finish_grow + 2 6 0.50 * cmp dword ptr [rsp + 8], 1 + 1 1 1.00 je .LBB6_2 + 1 5 0.50 * mov rax, qword ptr [rsp + 16] + 1 1 1.00 * mov qword ptr [rbx + 8], rax + 1 1 1.00 * mov qword ptr [rbx], r14 + 1 1 0.33 mov rsi, r13 + 1 1 1.00 jmp .LBB6_4 + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 12.00 12.00 10.00 13.00 11.00 11.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.49 0.51 push r15 + - - - - 1.00 - 0.51 0.49 push r14 + - - - - 1.00 - 0.50 0.50 push r13 + - - - - 1.00 - 0.50 0.50 push r12 + - - - - 1.00 - 0.50 0.50 push rbx + - - 0.01 0.99 - - - - sub rsp, 32 + - - - - - 1.00 - - mov rbx, rdi + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - - 0.50 0.50 mov r12, qword ptr [rdi + 16] + - - - 1.00 - - - - mov rcx, rax + - - - 0.99 - 0.01 - - sub rcx, r12 + - - - - - 1.00 - - cmp rsi, rcx + - - - - - 1.00 - - jbe .LBB6_3 + - - 0.01 0.98 - 0.01 - - mov r15, r12 + - - 0.99 0.01 - - - - add r15, rsi + - - - - - 1.00 - - jae .LBB6_6 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - jmp .LBB6_5 + - - - - - - 0.50 0.50 mov rax, qword ptr [rbx + 8] + - - 1.00 - - - - - lea r15, [r12 + rsi] + - - 0.98 0.02 - - - - lea rcx, [r12 + 2*r12] + - - 0.99 0.01 - - - - lea rdi, [rax + 2*rcx] + - - - 1.00 - - - - add rsi, rsi + - - 0.99 0.01 - - - - lea rdx, [rsi + 2*rsi] + - - - - - - - - xor esi, esi + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + memset@GOTPCREL] + - - - - 1.00 - 0.50 0.50 mov qword ptr [rbx + 16], r15 + - - 0.01 0.99 - - - - mov al, 1 + - - 1.00 - - - - - add rsp, 32 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r12 + - - - - - - 0.50 0.50 pop r13 + - - - - - - 0.50 0.50 pop r14 + - - - - - - 0.50 0.50 pop r15 + - - - - - 1.00 - - ret + - - 1.00 - - - - - mov r13, rsi + - - 0.01 0.99 - - - - lea rcx, [rax + rax] + - - 0.99 0.01 - - - - cmp r15, rcx + - - 2.00 0.01 - 0.99 - - cmova rcx, r15 + - - 0.01 0.99 - - - - cmp rcx, 5 + - - 0.01 0.99 - - - - mov r14d, 4 + - - 1.00 0.01 - 0.99 - - cmovae r14, rcx + - - - - - - 0.50 0.50 mov rdx, qword ptr [rbx + 8] + - - 0.01 0.99 - - - - lea rdi, [rsp + 8] + - - - 1.00 - - - - mov rsi, rax + - - - 0.01 - 0.99 - - mov rcx, r14 + - - - - 1.00 1.00 0.50 0.50 call ::finish_grow + - - - 0.99 - 0.01 0.50 0.50 cmp dword ptr [rsp + 8], 1 + - - - - - 1.00 - - je .LBB6_2 + - - - - - - 0.50 0.50 mov rax, qword ptr [rsp + 16] + - - - - 1.00 - 0.49 0.51 mov qword ptr [rbx + 8], rax + - - - - 1.00 - 0.51 0.49 mov qword ptr [rbx], r14 + - - 0.99 0.01 - - - - mov rsi, r13 + - - - - - 1.00 - - jmp .LBB6_4 diff --git a/rust/zerocopy/benches/formats/coco_dynamic_padding.rs b/rust/zerocopy/benches/formats/coco_dynamic_padding.rs new file mode 100644 index 000000000000..e494bce67312 --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_dynamic_padding.rs @@ -0,0 +1,24 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +#[derive(FromBytes, KnownLayout, Immutable, SplitAt)] +#[repr(C, align(4))] +pub struct Packet { + magic_number: Magic, + milk: u8, + mug_size: u8, + temperature: [u8; 5], + marshmallows: [[u8; 3]], +} + +/// A packet begining with the magic number `0xC0C0`. +pub type CocoPacket = Packet; + +/// A packet beginning with any two initialized bytes. +pub type LocoPacket = Packet<[u8; 2]>; diff --git a/rust/zerocopy/benches/formats/coco_dynamic_size.rs b/rust/zerocopy/benches/formats/coco_dynamic_size.rs new file mode 100644 index 000000000000..59364638e66c --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_dynamic_size.rs @@ -0,0 +1,27 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable, IntoBytes)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +macro_rules! define_packet { + ($name: ident, $trait: ident, $leading_field: ty) => { + #[derive($trait, KnownLayout, Immutable, IntoBytes, SplitAt)] + #[repr(C, align(2))] + pub struct $name { + magic_number: $leading_field, + mug_size: u8, + temperature: u8, + marshmallows: [[u8; 2]], + } + }; +} + +/// Packet begins with bytes 0xC0C0. +define_packet!(CocoPacket, TryFromBytes, C0C0); + +/// Packet begins with any two bytes. +define_packet!(LocoPacket, FromBytes, [u8; 2]); diff --git a/rust/zerocopy/benches/formats/coco_static_size.rs b/rust/zerocopy/benches/formats/coco_static_size.rs new file mode 100644 index 000000000000..0839497e1748 --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_static_size.rs @@ -0,0 +1,27 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable, IntoBytes)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +macro_rules! define_packet { + ($name: ident, $trait: ident, $leading_field: ty) => { + #[derive($trait, KnownLayout, Immutable, IntoBytes)] + #[repr(C, align(2))] + pub struct $name { + magic_number: $leading_field, + mug_size: u8, + temperature: u8, + marshmallows: [u8; 2], + } + }; +} + +/// Packet begins with bytes 0xC0C0. +define_packet!(CocoPacket, TryFromBytes, C0C0); + +/// Packet begins with any two bytes. +define_packet!(LocoPacket, FromBytes, [u8; 2]); diff --git a/rust/zerocopy/benches/insert_vec_zeroed.rs b/rust/zerocopy/benches/insert_vec_zeroed.rs new file mode 100644 index 000000000000..a5d685c2b027 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.rs @@ -0,0 +1,13 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_insert_vec_zeroed( + v: &mut Vec, + position: usize, + additional: usize, +) -> Option<()> { + FromZeros::insert_vec_zeroed(v, position, additional).ok() +} diff --git a/rust/zerocopy/benches/insert_vec_zeroed.x86-64 b/rust/zerocopy/benches/insert_vec_zeroed.x86-64 new file mode 100644 index 000000000000..9db87403cb55 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.x86-64 @@ -0,0 +1,79 @@ +bench_insert_vec_zeroed: + push rbp + push r15 + push r14 + push r13 + push r12 + push rbx + sub rsp, 24 + mov r12, qword ptr [rdi + 16] + mov r13, r12 + sub r13, rsi + jb .LBB6_10 + mov rbx, rdi + mov rax, qword ptr [rdi] + mov rcx, rax + sub rcx, r12 + cmp rdx, rcx + jbe .LBB6_4 + add r12, rdx + jae .LBB6_7 +.LBB6_3: + xor eax, eax + jmp .LBB6_6 +.LBB6_4: + mov rax, qword ptr [rbx + 8] + add r12, rdx +.LBB6_5: + lea rcx, [rsi + 2*rsi] + lea r14, [rax + 2*rcx] + add rdx, rdx + lea r15, [rdx + 2*rdx] + lea rdi, [r14 + r15] + add r13, r13 + lea rdx, [2*r13] + add rdx, r13 + mov rsi, r14 + call qword ptr [rip + memmove@GOTPCREL] + mov rdi, r14 + xor esi, esi + mov rdx, r15 + call qword ptr [rip + memset@GOTPCREL] + mov qword ptr [rbx + 16], r12 + mov al, 1 +.LBB6_6: + add rsp, 24 + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + pop rbp + ret +.LBB6_7: + mov r15, rsi + mov rbp, rdx + lea rcx, [rax + rax] + cmp r12, rcx + cmova rcx, r12 + cmp rcx, 5 + mov r14d, 4 + cmovae r14, rcx + mov rdx, qword ptr [rbx + 8] + mov rdi, rsp + mov rsi, rax + mov rcx, r14 + call ::finish_grow + cmp dword ptr [rsp], 1 + je .LBB6_3 + mov rax, qword ptr [rsp + 8] + mov qword ptr [rbx + 8], rax + mov qword ptr [rbx], r14 + mov rdx, rbp + mov rsi, r15 + jmp .LBB6_5 +.LBB6_10: + lea rdi, [rip + .Lanon.HASH.1] + lea rdx, [rip + .Lanon.HASH.3] + mov esi, 37 + call qword ptr [rip + core::panicking::panic@GOTPCREL] diff --git a/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..665240667844 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca @@ -0,0 +1,183 @@ +Iterations: 100 +Instructions: 7200 +Total Cycles: 7648 +Total uOps: 9300 + +Dispatch Width: 4 +uOps Per Cycle: 1.22 +IPC: 0.94 +Block RThroughput: 23.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push rbp + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push r13 + 2 5 1.00 * push r12 + 2 5 1.00 * push rbx + 1 1 0.33 sub rsp, 24 + 1 5 0.50 * mov r12, qword ptr [rdi + 16] + 1 1 0.33 mov r13, r12 + 1 1 0.33 sub r13, rsi + 1 1 1.00 jb .LBB6_10 + 1 1 0.33 mov rbx, rdi + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 1 0.33 mov rcx, rax + 1 1 0.33 sub rcx, r12 + 1 1 0.33 cmp rdx, rcx + 1 1 1.00 jbe .LBB6_4 + 1 1 0.33 add r12, rdx + 1 1 1.00 jae .LBB6_7 + 1 0 0.25 xor eax, eax + 1 1 1.00 jmp .LBB6_6 + 1 5 0.50 * mov rax, qword ptr [rbx + 8] + 1 1 0.33 add r12, rdx + 1 1 0.50 lea rcx, [rsi + 2*rsi] + 1 1 0.50 lea r14, [rax + 2*rcx] + 1 1 0.33 add rdx, rdx + 1 1 0.50 lea r15, [rdx + 2*rdx] + 1 1 0.50 lea rdi, [r14 + r15] + 1 1 0.33 add r13, r13 + 1 1 0.50 lea rdx, [2*r13] + 1 1 0.33 add rdx, r13 + 1 1 0.33 mov rsi, r14 + 4 7 1.00 * call qword ptr [rip + memmove@GOTPCREL] + 1 1 0.33 mov rdi, r14 + 1 0 0.25 xor esi, esi + 1 1 0.33 mov rdx, r15 + 4 7 1.00 * call qword ptr [rip + memset@GOTPCREL] + 1 1 1.00 * mov qword ptr [rbx + 16], r12 + 1 1 0.33 mov al, 1 + 1 1 0.33 add rsp, 24 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r12 + 1 6 0.50 * pop r13 + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 6 0.50 * pop rbp + 1 1 1.00 U ret + 1 1 0.33 mov r15, rsi + 1 1 0.33 mov rbp, rdx + 1 1 0.50 lea rcx, [rax + rax] + 1 1 0.33 cmp r12, rcx + 3 3 1.00 cmova rcx, r12 + 1 1 0.33 cmp rcx, 5 + 1 1 0.33 mov r14d, 4 + 2 2 0.67 cmovae r14, rcx + 1 5 0.50 * mov rdx, qword ptr [rbx + 8] + 1 1 0.33 mov rdi, rsp + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rcx, r14 + 3 5 1.00 call ::finish_grow + 2 6 0.50 * cmp dword ptr [rsp], 1 + 1 1 1.00 je .LBB6_3 + 1 5 0.50 * mov rax, qword ptr [rsp + 8] + 1 1 1.00 * mov qword ptr [rbx + 8], rax + 1 1 1.00 * mov qword ptr [rbx], r14 + 1 1 0.33 mov rdx, rbp + 1 1 0.33 mov rsi, r15 + 1 1 1.00 jmp .LBB6_5 + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.1] + 1 1 0.50 lea rdx, [rip + .Lanon.HASH.3] + 1 1 0.33 mov esi, 37 + 4 7 1.00 * call qword ptr [rip + core::panicking::panic@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 17.02 16.50 13.00 19.48 14.00 14.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.98 0.02 push rbp + - - - - 1.00 - 0.02 0.98 push r15 + - - - - 1.00 - 0.99 0.01 push r14 + - - - - 1.00 - 0.01 0.99 push r13 + - - - - 1.00 - 0.99 0.01 push r12 + - - - - 1.00 - 0.01 0.99 push rbx + - - 0.49 0.51 - - - - sub rsp, 24 + - - - - - - 0.04 0.96 mov r12, qword ptr [rdi + 16] + - - 0.49 0.50 - 0.01 - - mov r13, r12 + - - 0.48 0.51 - 0.01 - - sub r13, rsi + - - - - - 1.00 - - jb .LBB6_10 + - - 0.49 0.49 - 0.02 - - mov rbx, rdi + - - - - - - 0.97 0.03 mov rax, qword ptr [rdi] + - - 0.51 0.49 - - - - mov rcx, rax + - - 0.49 0.02 - 0.49 - - sub rcx, r12 + - - 0.49 0.50 - 0.01 - - cmp rdx, rcx + - - - - - 1.00 - - jbe .LBB6_4 + - - 0.02 0.49 - 0.49 - - add r12, rdx + - - - - - 1.00 - - jae .LBB6_7 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - jmp .LBB6_6 + - - - - - - 0.97 0.03 mov rax, qword ptr [rbx + 8] + - - 0.51 0.49 - - - - add r12, rdx + - - 0.49 0.51 - - - - lea rcx, [rsi + 2*rsi] + - - 0.50 0.50 - - - - lea r14, [rax + 2*rcx] + - - 0.51 0.49 - - - - add rdx, rdx + - - 0.50 0.50 - - - - lea r15, [rdx + 2*rdx] + - - 0.49 0.51 - - - - lea rdi, [r14 + r15] + - - 0.50 0.49 - 0.01 - - add r13, r13 + - - 0.51 0.49 - - - - lea rdx, [2*r13] + - - 0.01 0.01 - 0.98 - - add rdx, r13 + - - 0.01 - - 0.99 - - mov rsi, r14 + - - - - 1.00 1.00 1.98 0.02 call qword ptr [rip + memmove@GOTPCREL] + - - 0.49 0.50 - 0.01 - - mov rdi, r14 + - - - - - - - - xor esi, esi + - - 0.50 0.50 - - - - mov rdx, r15 + - - - - 1.00 1.00 1.96 0.04 call qword ptr [rip + memset@GOTPCREL] + - - - - 1.00 - 0.01 0.99 mov qword ptr [rbx + 16], r12 + - - 0.50 - - 0.50 - - mov al, 1 + - - 0.51 0.49 - - - - add rsp, 24 + - - - - - - 0.02 0.98 pop rbx + - - - - - - 0.03 0.97 pop r12 + - - - - - - 0.03 0.97 pop r13 + - - - - - - 0.97 0.03 pop r14 + - - - - - - 0.03 0.97 pop r15 + - - - - - - 0.01 0.99 pop rbp + - - - - - 1.00 - - ret + - - 0.49 0.51 - - - - mov r15, rsi + - - 0.51 0.49 - - - - mov rbp, rdx + - - 0.49 0.51 - - - - lea rcx, [rax + rax] + - - 0.49 0.50 - 0.01 - - cmp r12, rcx + - - 1.04 0.50 - 1.46 - - cmova rcx, r12 + - - 0.49 0.49 - 0.02 - - cmp rcx, 5 + - - 0.50 - - 0.50 - - mov r14d, 4 + - - 0.50 0.51 - 0.99 - - cmovae r14, rcx + - - - - - - 0.97 0.03 mov rdx, qword ptr [rbx + 8] + - - - 0.51 - 0.49 - - mov rdi, rsp + - - 0.01 0.50 - 0.49 - - mov rsi, rax + - - 0.49 0.50 - 0.01 - - mov rcx, r14 + - - - - 1.00 1.00 0.99 0.01 call ::finish_grow + - - 0.51 0.49 - - 0.50 0.50 cmp dword ptr [rsp], 1 + - - - - - 1.00 - - je .LBB6_3 + - - - - - - 0.50 0.50 mov rax, qword ptr [rsp + 8] + - - - - 1.00 - 0.99 0.01 mov qword ptr [rbx + 8], rax + - - - - 1.00 - 0.01 0.99 mov qword ptr [rbx], r14 + - - 0.49 0.50 - 0.01 - - mov rdx, rbp + - - 0.50 0.01 - 0.49 - - mov rsi, r15 + - - - - - 1.00 - - jmp .LBB6_5 + - - 0.01 0.99 - - - - lea rdi, [rip + .Lanon.HASH.1] + - - 0.99 0.01 - - - - lea rdx, [rip + .Lanon.HASH.3] + - - 0.02 0.49 - 0.49 - - mov esi, 37 + - - - - 1.00 1.00 0.02 1.98 call qword ptr [rip + core::panicking::panic@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed.rs b/rust/zerocopy/benches/new_box_zeroed.rs new file mode 100644 index 000000000000..aa9a66cce353 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed() -> Option> { + FromZeros::new_box_zeroed().ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed.x86-64 b/rust/zerocopy/benches/new_box_zeroed.x86-64 new file mode 100644 index 000000000000..ef74ea5388ac --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.x86-64 @@ -0,0 +1,7 @@ +bench_new_box_zeroed: + push rax + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov edi, 6 + mov esi, 2 + pop rax + jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed.x86-64.mca new file mode 100644 index 000000000000..05afa7feb0b8 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.x86-64.mca @@ -0,0 +1,51 @@ +Iterations: 100 +Instructions: 600 +Total Cycles: 1197 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 0.92 +IPC: 0.50 +Block RThroughput: 2.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push rax + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov edi, 6 + 1 1 0.33 mov esi, 2 + 1 6 0.50 * pop rax + 2 6 1.00 * jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 2.00 2.01 2.07 2.93 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.93 0.07 push rax + - - - - 1.00 1.00 0.12 1.88 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - 0.99 - - 0.01 - - mov edi, 6 + - - - 1.00 - - - - mov esi, 2 + - - - - - - 0.94 0.06 pop rax + - - - - - 1.00 0.08 0.92 jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..0afde999bff8 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed_with_elems_dynamic_padding( + count: usize, +) -> Option> { + FromZeros::new_box_zeroed_with_elems(count).ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..22a8d048ce0f --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,24 @@ +bench_new_box_zeroed_with_elems_dynamic_padding: + push r14 + push rbx + push rax + mov rbx, rdi + movabs rax, 3074457345618258598 + cmp rdi, rax + ja .LBB5_1 + lea r14, [rbx + 2*rbx] + or r14, 3 + add r14, 9 + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 4 + mov rdi, r14 + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + jmp .LBB5_3 +.LBB5_1: + xor eax, eax +.LBB5_3: + mov rdx, rbx + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..e6efaeded476 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,81 @@ +Iterations: 100 +Instructions: 2100 +Total Cycles: 2990 +Total uOps: 3000 + +Dispatch Width: 4 +uOps Per Cycle: 1.00 +IPC: 0.70 +Block RThroughput: 7.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rdi + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdi, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r14, [rbx + 2*rbx] + 1 1 0.33 or r14, 3 + 1 1 0.33 add r14, 9 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 4 + 1 1 0.33 mov rdi, r14 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 1.00 jmp .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rbx + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.49 4.50 5.00 6.01 4.50 4.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.50 0.50 push r14 + - - - - 1.00 - 0.50 0.50 push rbx + - - - - 1.00 - 0.50 0.50 push rax + - - 0.49 0.50 - 0.01 - - mov rbx, rdi + - - 0.50 0.50 - - - - movabs rax, 3074457345618258598 + - - 0.50 0.50 - - - - cmp rdi, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.50 0.50 - - - - lea r14, [rbx + 2*rbx] + - - 0.50 0.50 - - - - or r14, 3 + - - 0.50 - - 0.50 - - add r14, 9 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - - 0.50 - 0.50 - - mov esi, 4 + - - 0.50 0.50 - - - - mov rdi, r14 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - - - - 1.00 - - jmp .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.51 0.49 - - - - mov rdx, rbx + - - 0.49 0.51 - - - - add rsp, 8 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..1b12ca220692 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed_with_elems_dynamic_size(count: usize) -> Option> { + FromZeros::new_box_zeroed_with_elems(count).ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..bff15e55ad9f --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_new_box_zeroed_with_elems_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rdi + movabs rax, 4611686018427387901 + cmp rdi, rax + ja .LBB5_1 + lea r14, [2*rbx + 4] + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 2 + mov rdi, r14 + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + jmp .LBB5_3 +.LBB5_1: + xor eax, eax +.LBB5_3: + mov rdx, rbx + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..153d36c01ce0 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2990 +Total uOps: 2800 + +Dispatch Width: 4 +uOps Per Cycle: 0.94 +IPC: 0.64 +Block RThroughput: 7.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rdi + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdi, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r14, [2*rbx + 4] + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 2 + 1 1 0.33 mov rdi, r14 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 1.00 jmp .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rbx + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.97 3.98 5.00 5.05 4.50 4.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.50 0.50 push r14 + - - - - 1.00 - 0.50 0.50 push rbx + - - - - 1.00 - 0.50 0.50 push rax + - - 0.05 0.94 - 0.01 - - mov rbx, rdi + - - 0.94 0.06 - - - - movabs rax, 4611686018427387901 + - - 0.06 0.94 - - - - cmp rdi, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.94 0.06 - - - - lea r14, [2*rbx + 4] + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - 0.98 0.02 - - - - mov esi, 2 + - - 0.02 0.94 - 0.04 - - mov rdi, r14 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - - - - 1.00 - - jmp .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.94 0.06 - - - - mov rdx, rbx + - - 0.04 0.96 - - - - add rsp, 8 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_vec_zeroed.rs b/rust/zerocopy/benches/new_vec_zeroed.rs new file mode 100644 index 000000000000..3d95b2b24d8d --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_vec_zeroed(len: usize) -> Option> { + FromZeros::new_vec_zeroed(len).ok() +} diff --git a/rust/zerocopy/benches/new_vec_zeroed.x86-64 b/rust/zerocopy/benches/new_vec_zeroed.x86-64 new file mode 100644 index 000000000000..b5c083aa0d36 --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.x86-64 @@ -0,0 +1,40 @@ +bench_new_vec_zeroed: + mov rax, rdi + movabs rcx, 1537228672809129301 + cmp rsi, rcx + ja .LBB5_5 + test rsi, rsi + je .LBB5_2 + push r15 + push r14 + push rbx + lea rcx, [rsi + rsi] + lea rbx, [rcx + 2*rcx] + mov r14, rax + mov r15, rsi + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 2 + mov rdi, rbx + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + mov rsi, r15 + mov rcx, rax + mov rax, r14 + test rcx, rcx + pop rbx + pop r14 + pop r15 + je .LBB5_5 + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rcx + mov qword ptr [rax + 16], rsi + ret +.LBB5_5: + movabs rcx, -9223372036854775808 + mov qword ptr [rax], rcx + ret +.LBB5_2: + mov ecx, 2 + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rcx + mov qword ptr [rax + 16], rsi + ret diff --git a/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..b4fb4544ec39 --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca @@ -0,0 +1,113 @@ +Iterations: 100 +Instructions: 3700 +Total Cycles: 3486 +Total uOps: 4600 + +Dispatch Width: 4 +uOps Per Cycle: 1.32 +IPC: 1.06 +Block RThroughput: 12.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 movabs rcx, 1537228672809129301 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 ja .LBB5_5 + 1 1 0.33 test rsi, rsi + 1 1 1.00 je .LBB5_2 + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 1 1 0.50 lea rcx, [rsi + rsi] + 1 1 0.50 lea rbx, [rcx + 2*rcx] + 1 1 0.33 mov r14, rax + 1 1 0.33 mov r15, rsi + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 2 + 1 1 0.33 mov rdi, rbx + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 0.33 mov rsi, r15 + 1 1 0.33 mov rcx, rax + 1 1 0.33 mov rax, r14 + 1 1 0.33 test rcx, rcx + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 1 1.00 je .LBB5_5 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rcx + 1 1 1.00 * mov qword ptr [rax + 16], rsi + 1 1 1.00 U ret + 1 1 0.33 movabs rcx, -9223372036854775808 + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + 1 1 0.33 mov ecx, 2 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rcx + 1 1 1.00 * mov qword ptr [rax + 16], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 6.99 12.00 10.02 8.00 9.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - mov rax, rdi + - - 0.98 0.02 - - - - movabs rcx, 1537228672809129301 + - - 0.02 0.98 - - - - cmp rsi, rcx + - - - - - 1.00 - - ja .LBB5_5 + - - 0.98 - - 0.02 - - test rsi, rsi + - - - - - 1.00 - - je .LBB5_2 + - - - - 1.00 - - 1.00 push r15 + - - - - 1.00 - 1.00 - push r14 + - - - - 1.00 - - 1.00 push rbx + - - - 1.00 - - - - lea rcx, [rsi + rsi] + - - - 1.00 - - - - lea rbx, [rcx + 2*rcx] + - - 1.00 - - - - - mov r14, rax + - - 1.00 - - - - - mov r15, rsi + - - - - 1.00 1.00 2.00 - call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - - 0.01 - 0.99 - - mov esi, 2 + - - 0.01 0.99 - - - - mov rdi, rbx + - - - - 1.00 1.00 - 2.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - 0.01 - - 0.99 - - mov rsi, r15 + - - 0.99 0.01 - - - - mov rcx, rax + - - - 0.99 - 0.01 - - mov rax, r14 + - - 0.99 0.01 - - - - test rcx, rcx + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - - - 1.00 pop r15 + - - - - - 1.00 - - je .LBB5_5 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rsi + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - movabs rcx, -9223372036854775808 + - - - - 1.00 - - 1.00 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret + - - 0.99 0.01 - - - - mov ecx, 2 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_zeroed.rs b/rust/zerocopy/benches/new_zeroed.rs new file mode 100644 index 000000000000..b49f62edb146 --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_zeroed() -> format::LocoPacket { + FromZeros::new_zeroed() +} diff --git a/rust/zerocopy/benches/new_zeroed.x86-64 b/rust/zerocopy/benches/new_zeroed.x86-64 new file mode 100644 index 000000000000..b4d305e41fff --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.x86-64 @@ -0,0 +1,3 @@ +bench_new_zeroed: + xor eax, eax + ret diff --git a/rust/zerocopy/benches/new_zeroed.x86-64.mca b/rust/zerocopy/benches/new_zeroed.x86-64.mca new file mode 100644 index 000000000000..44583ca3089f --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 103 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.94 +IPC: 1.94 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_bytes.rs b/rust/zerocopy/benches/read_from_bytes.rs new file mode 100644 index 000000000000..8a3baddad9cb --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_bytes_static_size(source: &[u8]) -> Option { + zerocopy::FromBytes::read_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/read_from_bytes.x86-64 b/rust/zerocopy/benches/read_from_bytes.x86-64 new file mode 100644 index 000000000000..9082d79f1fd5 --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.x86-64 @@ -0,0 +1,15 @@ +bench_read_from_bytes_static_size: + mov rcx, rsi + cmp rsi, 6 + jne .LBB5_2 + mov eax, dword ptr [rdi] + movzx ecx, word ptr [rdi + 4] + shl rcx, 32 + or rcx, rax +.LBB5_2: + shl rcx, 16 + inc rcx + xor eax, eax + cmp rsi, 6 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/read_from_bytes.x86-64.mca b/rust/zerocopy/benches/read_from_bytes.x86-64.mca new file mode 100644 index 000000000000..77e787c19032 --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 377 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.71 +IPC: 3.45 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rcx, rsi + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx ecx, word ptr [rdi + 4] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shl rcx, 16 + 1 1 0.33 inc rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.67 - 3.67 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.63 0.36 - 0.01 - - mov rcx, rsi + - - 0.05 0.05 - 0.90 - - cmp rsi, 6 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + 4] + - - 0.97 - - 0.03 - - shl rcx, 32 + - - 0.02 0.35 - 0.63 - - or rcx, rax + - - 0.98 - - 0.02 - - shl rcx, 16 + - - - 0.98 - 0.02 - - inc rcx + - - - - - - - - xor eax, eax + - - 0.03 0.93 - 0.04 - - cmp rsi, 6 + - - 0.98 1.00 - 0.02 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_prefix.rs b/rust/zerocopy/benches/read_from_prefix.rs new file mode 100644 index 000000000000..d49bf80ab785 --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_prefix_static_size(source: &[u8]) -> Option { + match zerocopy::FromBytes::read_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/read_from_prefix.x86-64 b/rust/zerocopy/benches/read_from_prefix.x86-64 new file mode 100644 index 000000000000..c75b06c0c22a --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.x86-64 @@ -0,0 +1,14 @@ +bench_read_from_prefix_static_size: + cmp rsi, 5 + jbe .LBB5_2 + mov eax, dword ptr [rdi] + movzx edi, word ptr [rdi + 4] + shl rdi, 32 + or rdi, rax +.LBB5_2: + shl rdi, 16 + inc rdi + xor eax, eax + cmp rsi, 6 + cmovae rax, rdi + ret diff --git a/rust/zerocopy/benches/read_from_prefix.x86-64.mca b/rust/zerocopy/benches/read_from_prefix.x86-64.mca new file mode 100644 index 000000000000..04e76cdd07ee --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 905 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 1.44 +IPC: 1.33 +Block RThroughput: 3.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rsi, 5 + 1 1 1.00 jbe .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx edi, word ptr [rdi + 4] + 1 1 0.50 shl rdi, 32 + 1 1 0.33 or rdi, rax + 1 1 0.50 shl rdi, 16 + 1 1 0.33 inc rdi + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmovae rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.32 3.32 - 3.36 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.05 0.94 - 0.01 - - cmp rsi, 5 + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx edi, word ptr [rdi + 4] + - - 0.71 - - 0.29 - - shl rdi, 32 + - - - 0.64 - 0.36 - - or rdi, rax + - - 1.00 - - - - - shl rdi, 16 + - - 0.31 0.40 - 0.29 - - inc rdi + - - - - - - - - xor eax, eax + - - 0.34 0.35 - 0.31 - - cmp rsi, 6 + - - 0.91 0.99 - 0.10 - - cmovae rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_suffix.rs b/rust/zerocopy/benches/read_from_suffix.rs new file mode 100644 index 000000000000..4eaadb0d4dff --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_suffix_static_size(source: &[u8]) -> Option { + match zerocopy::FromBytes::read_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/read_from_suffix.x86-64 b/rust/zerocopy/benches/read_from_suffix.x86-64 new file mode 100644 index 000000000000..5cff2a0e2f36 --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.x86-64 @@ -0,0 +1,15 @@ +bench_read_from_suffix_static_size: + mov rcx, rsi + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi + rsi - 6] + movzx ecx, word ptr [rdi + rsi - 2] + shl rcx, 32 + or rcx, rax +.LBB5_2: + shl rcx, 16 + inc rcx + xor eax, eax + cmp rsi, 6 + cmovae rax, rcx + ret diff --git a/rust/zerocopy/benches/read_from_suffix.x86-64.mca b/rust/zerocopy/benches/read_from_suffix.x86-64.mca new file mode 100644 index 000000000000..0107de89562a --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 377 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.71 +IPC: 3.45 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rcx, rsi + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi + rsi - 6] + 1 5 0.50 * movzx ecx, word ptr [rdi + rsi - 2] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shl rcx, 16 + 1 1 0.33 inc rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmovae rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.67 - 3.67 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.63 0.36 - 0.01 - - mov rcx, rsi + - - 0.05 0.05 - 0.90 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi + rsi - 6] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + rsi - 2] + - - 0.97 - - 0.03 - - shl rcx, 32 + - - 0.02 0.35 - 0.63 - - or rcx, rax + - - 0.98 - - 0.02 - - shl rcx, 16 + - - - 0.98 - 0.02 - - inc rcx + - - - - - - - - xor eax, eax + - - 0.03 0.93 - 0.04 - - cmp rsi, 6 + - - 0.98 1.00 - 0.02 - - cmovae rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs new file mode 100644 index 000000000000..29708df55b45 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 new file mode 100644 index 000000000000..e844a4608fac --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_bytes_dynamic_padding: + test dil, 3 + jne .LBB5_3 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jb .LBB5_3 + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rsi, rax + je .LBB5_4 +.LBB5_3: + xor edi, edi + mov rdx, rsi +.LBB5_4: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..423ed38ba28d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 645 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.10 +IPC: 2.95 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 test dil, 3 + 1 1 1.00 jne .LBB5_3 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jb .LBB5_3 + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rsi, rax + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.32 6.33 - 6.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.64 0.35 - 0.01 - - test dil, 3 + - - - - - 1.00 - - jne .LBB5_3 + - - 0.34 0.65 - 0.01 - - movabs rax, 9223372036854775804 + - - 0.35 0.65 - - - - and rax, rsi + - - 0.33 0.34 - 0.33 - - cmp rax, 9 + - - - - - 1.00 - - jb .LBB5_3 + - - 0.35 - - 0.65 - - add rax, -9 + - - 0.97 0.01 - 0.02 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.99 - - 0.01 - - shr rdx + - - 0.33 0.67 - - - - lea rax, [rdx + 2*rdx] + - - 0.34 0.66 - - - - or rax, 3 + - - 0.33 0.66 - 0.01 - - add rax, 9 + - - 0.01 0.99 - - - - cmp rsi, rax + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor edi, edi + - - 0.32 0.01 - 0.67 - - mov rdx, rsi + - - 0.02 0.34 - 0.64 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs new file mode 100644 index 000000000000..4eb4f970e365 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..cc905b76c06f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 @@ -0,0 +1,20 @@ +bench_ref_from_bytes_dynamic_size: + mov rdx, rsi + cmp rsi, 4 + setb al + or al, dil + test al, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rcx, [rdx - 4] + mov rsi, rcx + and rsi, -2 + add rsi, 4 + shr rcx + xor eax, eax + cmp rdx, rsi + cmove rdx, rcx + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..68aea583e401 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,75 @@ +Iterations: 100 +Instructions: 1800 +Total Cycles: 704 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 2.84 +IPC: 2.56 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb al + 1 1 0.33 or al, dil + 1 1 0.33 test al, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rdx - 4] + 1 1 0.33 mov rsi, rcx + 1 1 0.33 and rsi, -2 + 1 1 0.33 add rsi, 4 + 1 1 0.50 shr rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rdx, rsi + 2 2 0.67 cmove rdx, rcx + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.97 5.98 - 6.05 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.97 0.01 - 0.02 - - mov rdx, rsi + - - 0.01 0.02 - 0.97 - - cmp rsi, 4 + - - 0.03 - - 0.97 - - setb al + - - 0.01 0.02 - 0.97 - - or al, dil + - - - 0.98 - 0.02 - - test al, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.98 0.02 - - - - lea rcx, [rdx - 4] + - - 0.01 0.99 - - - - mov rsi, rcx + - - - 0.98 - 0.02 - - and rsi, -2 + - - 0.98 0.01 - 0.01 - - add rsi, 4 + - - 0.99 - - 0.01 - - shr rcx + - - - - - - - - xor eax, eax + - - 0.02 0.97 - 0.01 - - cmp rdx, rsi + - - 0.99 0.99 - 0.02 - - cmove rdx, rcx + - - 0.98 0.99 - 0.03 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.rs b/rust/zerocopy/benches/ref_from_bytes_static_size.rs new file mode 100644 index 000000000000..3742bba0780f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 new file mode 100644 index 000000000000..2c8da68c8b5b --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 @@ -0,0 +1,8 @@ +bench_ref_from_bytes_static_size: + mov ecx, edi + and ecx, 1 + xor rsi, 6 + xor eax, eax + or rsi, rcx + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..832697801ee2 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca @@ -0,0 +1,53 @@ +Iterations: 100 +Instructions: 700 +Total Cycles: 240 +Total uOps: 800 + +Dispatch Width: 4 +uOps Per Cycle: 3.33 +IPC: 2.92 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov ecx, edi + 1 1 0.33 and ecx, 1 + 1 1 0.33 xor rsi, 6 + 1 0 0.25 xor eax, eax + 1 1 0.33 or rsi, rcx + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.33 2.33 - 2.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - mov ecx, edi + - - 0.02 0.66 - 0.32 - - and ecx, 1 + - - 0.33 0.66 - 0.01 - - xor rsi, 6 + - - - - - - - - xor eax, eax + - - 0.98 0.02 - - - - or rsi, rcx + - - 0.99 0.01 - 1.00 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..b4fea4aee51d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..d579b3faefe7 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,19 @@ +bench_ref_from_bytes_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + seta cl + mov rax, rdi + test al, 3 + setne dil + or dil, cl + jne .LBB5_2 + lea rcx, [rdx + 2*rdx] + or rcx, 3 + add rcx, 9 + cmp rsi, rcx + je .LBB5_3 +.LBB5_2: + xor eax, eax + mov rdx, rsi +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..ea2d83dbd17a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 539 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.15 +IPC: 2.97 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 test al, 3 + 1 1 0.50 setne dil + 1 1 0.33 or dil, cl + 1 1 1.00 jne .LBB5_2 + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 or rcx, 3 + 1 1 0.33 add rcx, 9 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.33 5.32 - 5.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - 1.98 - - 0.02 - - seta cl + - - 0.02 0.98 - - - - mov rax, rdi + - - - 0.67 - 0.33 - - test al, 3 + - - 0.67 - - 0.33 - - setne dil + - - 0.99 - - 0.01 - - or dil, cl + - - - - - 1.00 - - jne .LBB5_2 + - - 0.01 0.99 - - - - lea rcx, [rdx + 2*rdx] + - - - 0.01 - 0.99 - - or rcx, 3 + - - 0.65 0.02 - 0.33 - - add rcx, 9 + - - 0.99 0.01 - - - - cmp rsi, rcx + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.01 0.66 - 0.33 - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..9d33a7c31bc3 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..3d8d15b7f6c1 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 @@ -0,0 +1,16 @@ +bench_ref_from_bytes_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + seta cl + mov rax, rdi + or dil, cl + test dil, 1 + jne .LBB5_2 + lea rcx, [2*rdx + 4] + cmp rsi, rcx + je .LBB5_3 +.LBB5_2: + xor eax, eax + mov rdx, rsi +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..602179f3c903 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 439 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.19 +IPC: 2.96 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 or dil, cl + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_2 + 1 1 0.50 lea rcx, [2*rdx + 4] + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.32 4.33 - 4.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.33 0.67 - - - - cmp rdx, rax + - - 1.98 - - 0.02 - - seta cl + - - 0.01 0.99 - - - - mov rax, rdi + - - 1.00 - - - - - or dil, cl + - - 0.99 0.01 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - - 1.00 - - - - lea rcx, [2*rdx + 4] + - - 0.01 - - 0.99 - - cmp rsi, rcx + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - 0.67 - 0.33 - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs new file mode 100644 index 000000000000..53c707b88256 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..a58592a24503 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_prefix_dynamic_padding: + xor edx, edx + mov eax, 0 + test dil, 3 + je .LBB5_1 + ret +.LBB5_1: + movabs rax, 9223372036854775804 + and rsi, rax + cmp rsi, 9 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -9 + movabs rcx, -6148914691236517205 + mov rax, rsi + mul rcx + shr rdx + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..62ea4babaf28 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 608 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.13 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_1 + 1 1 1.00 U ret + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rsi, rax + 1 1 0.33 cmp rsi, 9 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov rax, rsi + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.00 6.00 - 6.00 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.01 - 0.01 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_1 + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - movabs rax, 9223372036854775804 + - - - 1.00 - - - - and rsi, rax + - - - 1.00 - - - - cmp rsi, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - 1.00 - - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.02 0.02 - 0.96 - - add rsi, -9 + - - 0.99 0.01 - - - - movabs rcx, -6148914691236517205 + - - 0.01 0.99 - - - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 1.00 - - - - - shr rdx + - - 0.98 - - 0.02 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs new file mode 100644 index 000000000000..a3f26f6b4e6c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..fe6332c9100c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 @@ -0,0 +1,17 @@ +bench_ref_from_prefix_dynamic_size: + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + cmp rsi, 4 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -4 + shr rsi + mov rdx, rsi + mov rax, rdi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..3900a5946138 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 405 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.46 +IPC: 3.46 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 cmp rsi, 4 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -4 + 1 1 0.50 shr rsi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.99 3.99 - 4.02 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.02 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.02 0.98 - - - - cmp rsi, 4 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.98 0.01 - 0.01 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - add rsi, -4 + - - 1.00 - - - - - shr rsi + - - - 1.00 - - - - mov rdx, rsi + - - 0.99 0.01 - - - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.rs b/rust/zerocopy/benches/ref_from_prefix_static_size.rs new file mode 100644 index 000000000000..834fa3928611 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 new file mode 100644 index 000000000000..7c1bf45bb6c2 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 @@ -0,0 +1,8 @@ +bench_ref_from_prefix_static_size: + xor eax, eax + cmp rsi, 6 + mov rcx, rdi + cmovb rcx, rax + test dil, 1 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..9691b88fe03a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca @@ -0,0 +1,53 @@ +Iterations: 100 +Instructions: 700 +Total Cycles: 274 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.28 +IPC: 2.55 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 1 1 0.33 mov rcx, rdi + 2 2 0.67 cmovb rcx, rax + 1 1 0.33 test dil, 1 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.66 2.67 - 2.67 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - - 0.01 - 0.99 - - cmp rsi, 6 + - - 0.01 0.67 - 0.32 - - mov rcx, rdi + - - 1.00 0.99 - 0.01 - - cmovb rcx, rax + - - 0.66 0.01 - 0.33 - - test dil, 1 + - - 0.99 0.99 - 0.02 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..55d495e00c59 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..5b31277bdebe --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_ref_from_prefix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + xor ecx, ecx + mov eax, 0 + test dil, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_1: + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + xor r8d, r8d + cmp rax, rsi + mov ecx, 1 + cmovbe rcx, rdx + cmova rdi, r8 + mov rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..2f212ec6d03b --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 807 +Total uOps: 2700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.85 +Block RThroughput: 6.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 0 0.25 xor r8d, r8d + 1 1 0.33 cmp rax, rsi + 1 1 0.33 mov ecx, 1 + 3 3 1.00 cmovbe rcx, rdx + 3 3 1.00 cmova rdi, r8 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.99 7.99 - 8.02 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.47 0.52 - 0.01 - - movabs rax, 3074457345618258598 + - - 0.94 0.01 - 0.05 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - - - - - - xor ecx, ecx + - - 0.03 0.97 - - - - mov eax, 0 + - - 0.01 0.52 - 0.47 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.03 0.51 - 0.46 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.04 0.96 - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - lea rax, [rdx + 2*rdx] + - - 0.52 0.48 - - - - or rax, 3 + - - 0.51 0.49 - - - - add rax, 9 + - - - - - - - - xor r8d, r8d + - - 0.97 0.03 - - - - cmp rax, rsi + - - 0.01 0.99 - - - - mov ecx, 1 + - - 1.04 0.97 - 0.99 - - cmovbe rcx, rdx + - - 1.44 0.54 - 1.02 - - cmova rdi, r8 + - - 0.97 0.01 - 0.02 - - mov rax, rdi + - - 1.00 - - - - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..e9663c721ece --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..069fd4859c74 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_prefix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + mov rcx, rdx + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + lea rax, [2*rcx + 4] + xor r8d, r8d + cmp rax, rsi + mov edx, 1 + cmovbe rdx, rcx + cmova rdi, r8 + mov rax, rdi +.LBB5_4: + ret +.LBB5_1: + mov edx, 1 + xor eax, eax + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6f227264066d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 672 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.42 +IPC: 2.83 +Block RThroughput: 5.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.33 mov rcx, rdx + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 lea rax, [2*rcx + 4] + 1 0 0.25 xor r8d, r8d + 1 1 0.33 cmp rax, rsi + 1 1 0.33 mov edx, 1 + 3 3 1.00 cmovbe rdx, rcx + 3 3 1.00 cmova rdi, r8 + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.66 6.66 - 6.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.37 0.63 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.63 0.37 - - - - mov rcx, rdx + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.02 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.01 0.99 - - - - lea rax, [2*rcx + 4] + - - - - - - - - xor r8d, r8d + - - 1.00 - - - - - cmp rax, rsi + - - - 0.67 - 0.33 - - mov edx, 1 + - - 0.73 0.98 - 1.29 - - cmovbe rdx, rcx + - - 1.60 0.36 - 1.04 - - cmova rdi, r8 + - - 0.99 0.01 - - - - mov rax, rdi + - - - - - 1.00 - - ret + - - 0.34 0.66 - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs new file mode 100644 index 000000000000..5a6ea3a33dde --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3e05f6023f38 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 @@ -0,0 +1,23 @@ +bench_ref_from_suffix_dynamic_padding: + lea eax, [rsi + rdi] + test al, 3 + jne .LBB5_1 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jae .LBB5_3 +.LBB5_1: + xor eax, eax + ret +.LBB5_3: + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + sub rsi, rax + or rax, -4 + add rsi, rdi + add rax, rsi + add rax, -8 + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..73599d5b6aab --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 682 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.08 +IPC: 2.93 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 test al, 3 + 1 1 1.00 jne .LBB5_1 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jae .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 sub rsi, rax + 1 1 0.33 or rax, -4 + 1 1 0.33 add rsi, rdi + 1 1 0.33 add rax, rsi + 1 1 0.33 add rax, -8 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.65 6.67 - 6.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.90 0.10 - - - - lea eax, [rsi + rdi] + - - 0.93 - - 0.07 - - test al, 3 + - - - - - 1.00 - - jne .LBB5_1 + - - 0.51 0.47 - 0.02 - - movabs rax, 9223372036854775804 + - - - - - 1.00 - - and rax, rsi + - - - 0.09 - 0.91 - - cmp rax, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.43 0.47 - 0.10 - - add rax, -9 + - - 0.42 0.39 - 0.19 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.69 - - 0.31 - - shr rdx + - - 0.54 0.46 - - - - lea rax, [rdx + 2*rdx] + - - 0.07 0.91 - 0.02 - - sub rsi, rax + - - 0.91 0.05 - 0.04 - - or rax, -4 + - - 0.08 0.90 - 0.02 - - add rsi, rdi + - - 0.09 0.91 - - - - add rax, rsi + - - 0.08 0.92 - - - - add rax, -8 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs new file mode 100644 index 000000000000..3437b14f404a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..bd4ace89836a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_ref_from_suffix_dynamic_size: + mov rdx, rsi + lea ecx, [rsi + rdi] + mov eax, edx + and eax, 1 + add rax, rdi + xor esi, esi + sub rdx, 4 + cmovb rax, rsi + shr rdx + test cl, 1 + cmovne rax, rsi + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..1398bcfe27ae --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 439 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.19 +IPC: 2.73 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.50 lea ecx, [rsi + rdi] + 1 1 0.33 mov eax, edx + 1 1 0.33 and eax, 1 + 1 1 0.33 add rax, rdi + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rdx, 4 + 2 2 0.67 cmovb rax, rsi + 1 1 0.50 shr rdx + 1 1 0.33 test cl, 1 + 2 2 0.67 cmovne rax, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.32 - 0.66 - - mov rdx, rsi + - - 0.32 0.68 - - - - lea ecx, [rsi + rdi] + - - 0.66 - - 0.34 - - mov eax, edx + - - 0.02 0.33 - 0.65 - - and eax, 1 + - - - 0.99 - 0.01 - - add rax, rdi + - - - - - - - - xor esi, esi + - - 0.65 - - 0.35 - - sub rdx, 4 + - - 1.00 1.00 - - - - cmovb rax, rsi + - - 0.66 - - 0.34 - - shr rdx + - - - 0.01 - 0.99 - - test cl, 1 + - - 1.00 1.00 - - - - cmovne rax, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.rs b/rust/zerocopy/benches/ref_from_suffix_static_size.rs new file mode 100644 index 000000000000..c8435d10d38c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 new file mode 100644 index 000000000000..9e90b9e2543f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 @@ -0,0 +1,13 @@ +bench_ref_from_suffix_static_size: + lea eax, [rsi + rdi] + cmp rsi, 6 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rax, [rdi + rsi] + add rax, -6 + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..ef5892647b81 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca @@ -0,0 +1,61 @@ +Iterations: 100 +Instructions: 1100 +Total Cycles: 338 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 3.25 +IPC: 3.25 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdi + rsi] + 1 1 0.33 add rax, -6 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.32 3.33 - 3.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.97 0.03 - - - - lea eax, [rsi + rdi] + - - 0.33 0.32 - 0.35 - - cmp rsi, 6 + - - 1.00 - - - - - setb cl + - - - 1.00 - - - - or cl, al + - - - 1.00 - - - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.34 0.66 - - - - lea rax, [rdi + rsi] + - - 0.68 0.32 - - - - add rax, -6 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..73d91cee5992 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..c3d10b5fc685 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,27 @@ +bench_ref_from_suffix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + add rdi, rsi + mov rcx, rdx + mov rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..92e6280bb4cc --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 706 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.26 +IPC: 3.26 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.33 add rdi, rsi + 1 1 0.33 mov rcx, rdx + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 7.00 - 7.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 3074457345618258598 + - - 0.01 0.50 - 0.49 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - 1.00 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.50 0.49 - 0.01 - - mov eax, 0 + - - 0.49 0.51 - - - - test r8b, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.51 0.49 - - - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.50 0.50 - - - - lea rax, [rdx + 2*rdx] + - - 1.00 - - - - - or rax, 3 + - - 1.00 - - - - - add rax, 9 + - - 0.99 0.01 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - - 1.00 - - - - mov edx, 1 + - - - - - 1.00 - - ret + - - 1.00 - - - - - add rdi, rsi + - - - 1.00 - - - - mov rcx, rdx + - - 0.99 0.01 - - - - mov rax, rdi + - - - 0.50 - 0.50 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..68a28baf55e6 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..bdca57192455 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,23 @@ +bench_ref_from_suffix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 1 + jne .LBB5_5 + lea rax, [2*rdx + 4] + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + add rdi, rsi + mov rcx, rdx + mov rax, rdi +.LBB5_5: + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6d9de0b3eb5c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 571 +Total uOps: 1900 + +Dispatch Width: 4 +uOps Per Cycle: 3.33 +IPC: 3.33 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rdx + 4] + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.33 add rdi, rsi + 1 1 0.33 mov rcx, rdx + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.66 5.66 - 5.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.66 0.33 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.01 0.99 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.99 0.01 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.33 0.33 - 0.34 - - mov eax, 0 + - - 0.33 0.34 - 0.33 - - test r8b, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.34 0.66 - - - - lea rax, [2*rdx + 4] + - - - 1.00 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 1.00 - - - - - mov edx, 1 + - - - - - 1.00 - - ret + - - - 1.00 - - - - add rdi, rsi + - - 1.00 - - - - - mov rcx, rdx + - - 0.32 0.68 - - - - mov rax, rdi + - - 0.68 0.32 - - - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.rs b/rust/zerocopy/benches/split_at_dynamic_padding.rs new file mode 100644 index 000000000000..bed90f60165e --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_at_dynamic_padding( + source: &format::CocoPacket, + len: usize, +) -> Option> { + source.split_at(len) +} diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 new file mode 100644 index 000000000000..6eaf5a004612 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 @@ -0,0 +1,12 @@ +bench_split_at_dynamic_padding: + mov rax, rdi + cmp rcx, rdx + jbe .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..19ab3414d77a --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 404 +Total uOps: 1000 + +Dispatch Width: 4 +uOps Per Cycle: 2.48 +IPC: 2.48 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rcx, rdx + 1 1 1.00 jbe .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 4.00 3.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - cmp rcx, rdx + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_dynamic_size.rs b/rust/zerocopy/benches/split_at_dynamic_size.rs new file mode 100644 index 000000000000..07a22ba2e5b5 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_at_dynamic_size( + source: &format::CocoPacket, + len: usize, +) -> Option> { + source.split_at(len) +} diff --git a/rust/zerocopy/benches/split_at_dynamic_size.x86-64 b/rust/zerocopy/benches/split_at_dynamic_size.x86-64 new file mode 100644 index 000000000000..8d81b98bdab1 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.x86-64 @@ -0,0 +1,12 @@ +bench_split_at_dynamic_size: + mov rax, rdi + cmp rcx, rdx + jbe .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..19ab3414d77a --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 404 +Total uOps: 1000 + +Dispatch Width: 4 +uOps Per Cycle: 2.48 +IPC: 2.48 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rcx, rdx + 1 1 1.00 jbe .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 4.00 3.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - cmp rcx, rdx + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs new file mode 100644 index 000000000000..3c147d3bdd39 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_at_unchecked_dynamic_padding( + source: &format::CocoPacket, + len: usize, +) -> Split<&format::CocoPacket> { + unsafe { source.split_at_unchecked(len) } +} diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 new file mode 100644 index 000000000000..74c3b52f63fa --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 @@ -0,0 +1,6 @@ +bench_split_at_unchecked_dynamic_padding: + mov rax, rdi + mov qword ptr [rdi], rsi + mov qword ptr [rdi + 8], rdx + mov qword ptr [rdi + 16], rcx + ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..e8c61591c086 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 303 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 1.65 +IPC: 1.65 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 1.00 * mov qword ptr [rdi + 8], rdx + 1 1 1.00 * mov qword ptr [rdi + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 3.00 1.01 1.50 1.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi], rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 8], rdx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs new file mode 100644 index 000000000000..b1aa1dfb35be --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_at_unchecked_dynamic_size( + source: &format::CocoPacket, + len: usize, +) -> Split<&format::CocoPacket> { + unsafe { source.split_at_unchecked(len) } +} diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 new file mode 100644 index 000000000000..56671d1ee8dc --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 @@ -0,0 +1,6 @@ +bench_split_at_unchecked_dynamic_size: + mov rax, rdi + mov qword ptr [rdi], rsi + mov qword ptr [rdi + 8], rdx + mov qword ptr [rdi + 16], rcx + ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..e8c61591c086 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 303 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 1.65 +IPC: 1.65 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 1.00 * mov qword ptr [rdi + 8], rdx + 1 1 1.00 * mov qword ptr [rdi + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 3.00 1.01 1.50 1.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi], rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 8], rdx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs new file mode 100644 index 000000000000..b86ad3614bf7 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_immutable_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 3]]) { + split.via_immutable() +} diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 new file mode 100644 index 000000000000..dac183428b31 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 @@ -0,0 +1,14 @@ +bench_split_via_immutable_dynamic_padding: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rsi + 2*rsi] + add rdi, rcx + add rdi, 9 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..6ab4e838767e --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 510 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 2.55 +IPC: 2.55 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rsi + 2*rsi] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.00 2.00 4.00 2.00 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.03 0.93 - 0.04 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 1.00 - mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.93 0.07 - - - - lea rdi, [rsi + 2*rsi] + - - 0.05 0.02 - 0.93 - - add rdi, rcx + - - 0.49 0.49 - 0.02 - - add rdi, 9 + - - 0.50 0.49 - 0.01 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.49 0.51 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs b/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs new file mode 100644 index 000000000000..7d115caa3c00 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_immutable_dynamic_size( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 2]]) { + split.via_immutable() +} diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 new file mode 100644 index 000000000000..58f6b09fc9d2 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_immutable_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs new file mode 100644 index 000000000000..edba7bf34ba8 --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_runtime_check_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> Option<(&format::CocoPacket, &[[u8; 3]])> { + split.via_runtime_check().ok() +} diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 new file mode 100644 index 000000000000..03684cb09f8f --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_split_via_runtime_check_dynamic_padding: + mov rax, rdi + mov rdx, qword ptr [rsi + 16] + mov ecx, edx + and ecx, 3 + cmp ecx, 1 + jne .LBB5_1 + mov rcx, qword ptr [rsi] + mov rsi, qword ptr [rsi + 8] + lea rdi, [rdx + 2*rdx] + add rdi, rcx + add rdi, 9 + sub rsi, rdx + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rsi + mov qword ptr [rax], rcx + ret +.LBB5_1: + xor ecx, ecx + mov qword ptr [rax], rcx + ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..5034ab0583bd --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 708 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 2.82 +IPC: 2.82 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rdx, qword ptr [rsi + 16] + 1 1 0.33 mov ecx, edx + 1 1 0.33 and ecx, 3 + 1 1 0.33 cmp ecx, 1 + 1 1 1.00 jne .LBB5_1 + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rsi, qword ptr [rsi + 8] + 1 1 0.50 lea rdi, [rdx + 2*rdx] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rsi, rdx + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + 1 0 0.25 xor ecx, ecx + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.00 3.02 5.00 4.98 4.00 4.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - mov rax, rdi + - - - - - - - 1.00 mov rdx, qword ptr [rsi + 16] + - - 0.99 0.01 - - - - mov ecx, edx + - - 0.99 0.01 - - - - and ecx, 3 + - - 0.97 0.03 - - - - cmp ecx, 1 + - - - - - 1.00 - - jne .LBB5_1 + - - - - - - 1.00 - mov rcx, qword ptr [rsi] + - - - - - - 0.99 0.01 mov rsi, qword ptr [rsi + 8] + - - 0.01 0.99 - - - - lea rdi, [rdx + 2*rdx] + - - - 0.96 - 0.04 - - add rdi, rcx + - - 0.03 - - 0.97 - - add rdi, 9 + - - 0.01 0.03 - 0.96 - - sub rsi, rdx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.01 0.99 mov qword ptr [rax + 24], rsi + - - - - 1.00 - 0.99 0.01 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret + - - - - - - - - xor ecx, ecx + - - - - 1.00 - 0.01 0.99 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs new file mode 100644 index 000000000000..ce22de909aca --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_runtime_check_dynamic_size( + split: Split<&format::CocoPacket>, +) -> Option<(&format::CocoPacket, &[[u8; 2]])> { + split.via_runtime_check().ok() +} diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 new file mode 100644 index 000000000000..e54276bd85e8 --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_runtime_check_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs new file mode 100644 index 000000000000..21d74dba176f --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_via_unchecked_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 3]]) { + unsafe { split.via_unchecked() } +} diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3c2c4ec9f6ed --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 @@ -0,0 +1,14 @@ +bench_split_via_unchecked_dynamic_padding: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rsi + 2*rsi] + add rdi, rcx + add rdi, 9 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..6ab4e838767e --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 510 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 2.55 +IPC: 2.55 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rsi + 2*rsi] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.00 2.00 4.00 2.00 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.03 0.93 - 0.04 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 1.00 - mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.93 0.07 - - - - lea rdi, [rsi + 2*rsi] + - - 0.05 0.02 - 0.93 - - add rdi, rcx + - - 0.49 0.49 - 0.02 - - add rdi, 9 + - - 0.50 0.49 - 0.01 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.49 0.51 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs new file mode 100644 index 000000000000..824e22d67a7b --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_via_unchecked_dynamic_size( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 2]]) { + unsafe { split.via_unchecked() } +} diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 new file mode 100644 index 000000000000..1e31268edc7f --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_unchecked_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute.rs b/rust/zerocopy/benches/transmute.rs new file mode 100644 index 000000000000..e60bfb252f5c --- /dev/null +++ b/rust/zerocopy/benches/transmute.rs @@ -0,0 +1,16 @@ +use zerocopy::Unalign; +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C)] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_transmute(source: MinimalViableSource) -> Unalign { + zerocopy::transmute!(source) +} diff --git a/rust/zerocopy/benches/transmute.x86-64 b/rust/zerocopy/benches/transmute.x86-64 new file mode 100644 index 000000000000..a4c6299d1112 --- /dev/null +++ b/rust/zerocopy/benches/transmute.x86-64 @@ -0,0 +1,3 @@ +bench_transmute: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/transmute.x86-64.mca b/rust/zerocopy/benches/transmute.x86-64.mca new file mode 100644 index 000000000000..f297729c6523 --- /dev/null +++ b/rust/zerocopy/benches/transmute.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 104 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.92 +IPC: 1.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.rs b/rust/zerocopy/benches/transmute_ref_dynamic_size.rs new file mode 100644 index 000000000000..825f0f2bed24 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.rs @@ -0,0 +1,16 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + header: [u8; 6], + trailer: [[u8; 2]], +} + +#[unsafe(no_mangle)] +fn bench_transmute_ref_dynamic_size(source: &MinimalViableSource) -> &format::LocoPacket { + zerocopy::transmute_ref!(source) +} diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 new file mode 100644 index 000000000000..80a0f5906610 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 @@ -0,0 +1,4 @@ +bench_transmute_ref_dynamic_size: + mov rax, rdi + lea rdx, [rsi + 1] + ret diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ef1bcdca5ea3 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 104 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.88 +IPC: 2.88 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdx, [rsi + 1] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - lea rdx, [rsi + 1] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute_ref_static_size.rs b/rust/zerocopy/benches/transmute_ref_static_size.rs new file mode 100644 index 000000000000..a6db611fde0d --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.rs @@ -0,0 +1,15 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_transmute_ref_static_size(source: &MinimalViableSource) -> &format::LocoPacket { + zerocopy::transmute_ref!(source) +} diff --git a/rust/zerocopy/benches/transmute_ref_static_size.x86-64 b/rust/zerocopy/benches/transmute_ref_static_size.x86-64 new file mode 100644 index 000000000000..7a9229c129b0 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.x86-64 @@ -0,0 +1,3 @@ +bench_transmute_ref_static_size: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca b/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca new file mode 100644 index 000000000000..f297729c6523 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 104 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.92 +IPC: 1.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_bytes.rs b/rust/zerocopy/benches/try_read_from_bytes.rs new file mode 100644 index 000000000000..f8384b32f518 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_bytes_static_size(source: &[u8]) -> Option { + zerocopy::TryFromBytes::try_read_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_read_from_bytes.x86-64 b/rust/zerocopy/benches/try_read_from_bytes.x86-64 new file mode 100644 index 000000000000..08088a08fd85 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.x86-64 @@ -0,0 +1,23 @@ +bench_try_read_from_bytes_static_size: + mov ax, -16191 + cmp rsi, 6 + jne .LBB5_1 + mov ecx, dword ptr [rdi] + movzx edx, cx + cmp edx, 49344 + jne .LBB5_4 + movzx eax, word ptr [rdi + 4] + shl rax, 32 + or rcx, rax + shr rcx, 16 + mov ax, -16192 +.LBB5_4: + shl rcx, 16 + movzx eax, ax + or rax, rcx + ret +.LBB5_1: + shl rcx, 16 + movzx eax, ax + or rax, rcx + ret diff --git a/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca b/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca new file mode 100644 index 000000000000..385e6a480253 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 608 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.29 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov ax, -16191 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jne .LBB5_1 + 1 5 0.50 * mov ecx, dword ptr [rdi] + 1 1 0.33 movzx edx, cx + 1 1 0.33 cmp edx, 49344 + 1 1 1.00 jne .LBB5_4 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 0.50 shl rax, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shr rcx, 16 + 1 1 0.33 mov ax, -16192 + 1 1 0.50 shl rcx, 16 + 1 1 0.33 movzx eax, ax + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + 1 1 0.50 shl rcx, 16 + 1 1 0.33 movzx eax, ax + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.99 5.99 - 6.02 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - mov ax, -16191 + - - - 0.01 - 0.99 - - cmp rsi, 6 + - - - - - 1.00 - - jne .LBB5_1 + - - - - - - - 1.00 mov ecx, dword ptr [rdi] + - - 0.98 - - 0.02 - - movzx edx, cx + - - 0.99 0.01 - - - - cmp edx, 49344 + - - - - - 1.00 - - jne .LBB5_4 + - - - - - - 1.00 - movzx eax, word ptr [rdi + 4] + - - 0.01 - - 0.99 - - shl rax, 32 + - - 0.02 0.98 - - - - or rcx, rax + - - 1.00 - - - - - shr rcx, 16 + - - 0.99 0.01 - - - - mov ax, -16192 + - - 1.00 - - - - - shl rcx, 16 + - - - 1.00 - - - - movzx eax, ax + - - - 1.00 - - - - or rax, rcx + - - - - - 1.00 - - ret + - - 1.00 - - - - - shl rcx, 16 + - - - 1.00 - - - - movzx eax, ax + - - - 0.99 - 0.01 - - or rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_prefix.rs b/rust/zerocopy/benches/try_read_from_prefix.rs new file mode 100644 index 000000000000..fabbac3bd7d9 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_prefix_static_size(source: &[u8]) -> Option { + match zerocopy::TryFromBytes::try_read_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_read_from_prefix.x86-64 b/rust/zerocopy/benches/try_read_from_prefix.x86-64 new file mode 100644 index 000000000000..d3e1edc3ea73 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.x86-64 @@ -0,0 +1,16 @@ +bench_try_read_from_prefix_static_size: + mov eax, 49345 + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi] + movzx ecx, word ptr [rdi + 4] + shl rcx, 32 + or rcx, rax + movzx eax, cx + and rcx, -65536 + or rcx, 49344 + cmp eax, 49344 + mov eax, 49345 + cmove rax, rcx +.LBB5_2: + ret diff --git a/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca b/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca new file mode 100644 index 000000000000..40401d89e83d --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 442 +Total uOps: 1500 + +Dispatch Width: 4 +uOps Per Cycle: 3.39 +IPC: 3.17 +Block RThroughput: 3.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov eax, 49345 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx ecx, word ptr [rdi + 4] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.33 movzx eax, cx + 1 1 0.33 and rcx, -65536 + 1 1 0.33 or rcx, 49344 + 1 1 0.33 cmp eax, 49344 + 1 1 0.33 mov eax, 49345 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.65 0.01 - 0.34 - - mov eax, 49345 + - - 0.01 0.33 - 0.66 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + 4] + - - 0.65 - - 0.35 - - shl rcx, 32 + - - - 0.67 - 0.33 - - or rcx, rax + - - 0.01 0.99 - - - - movzx eax, cx + - - 0.99 0.01 - - - - and rcx, -65536 + - - 0.01 0.99 - - - - or rcx, 49344 + - - 0.99 0.01 - - - - cmp eax, 49344 + - - 0.02 0.33 - 0.65 - - mov eax, 49345 + - - 1.00 0.99 - 0.01 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_suffix.rs b/rust/zerocopy/benches/try_read_from_suffix.rs new file mode 100644 index 000000000000..1e961647b3f3 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_suffix_static_size(source: &[u8]) -> Option { + match zerocopy::TryFromBytes::try_read_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_read_from_suffix.x86-64 b/rust/zerocopy/benches/try_read_from_suffix.x86-64 new file mode 100644 index 000000000000..095e326f0467 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.x86-64 @@ -0,0 +1,18 @@ +bench_try_read_from_suffix_static_size: + mov eax, 49345 + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi + rsi - 6] + movzx ecx, word ptr [rdi + rsi - 2] + shl rcx, 32 + or rcx, rax + movzx edx, cx + xor eax, eax + cmp edx, 49344 + cmovne rcx, rsi + sete al + and rcx, -65536 + xor rax, 49345 + or rax, rcx +.LBB5_2: + ret diff --git a/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca b/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca new file mode 100644 index 000000000000..d3eaadbb8a81 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 478 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.56 +IPC: 3.35 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov eax, 49345 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi + rsi - 6] + 1 5 0.50 * movzx ecx, word ptr [rdi + rsi - 2] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.33 movzx edx, cx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp edx, 49344 + 2 2 0.67 cmovne rcx, rsi + 1 1 0.50 sete al + 1 1 0.33 and rcx, -65536 + 1 1 0.33 xor rax, 49345 + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.66 4.66 - 4.68 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.01 - 0.67 - - mov eax, 49345 + - - 0.62 0.02 - 0.36 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi + rsi - 6] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + rsi - 2] + - - 0.37 - - 0.63 - - shl rcx, 32 + - - 0.99 0.01 - - - - or rcx, rax + - - 1.00 - - - - - movzx edx, cx + - - - - - - - - xor eax, eax + - - 0.35 0.64 - 0.01 - - cmp edx, 49344 + - - 1.00 1.00 - - - - cmovne rcx, rsi + - - - - - 1.00 - - sete al + - - 0.01 0.99 - - - - and rcx, -65536 + - - - 1.00 - - - - xor rax, 49345 + - - - 0.99 - 0.01 - - or rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs new file mode 100644 index 000000000000..126009cd71d5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 new file mode 100644 index 000000000000..217c5fc61796 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 @@ -0,0 +1,24 @@ +bench_try_ref_from_bytes_dynamic_padding: + test dil, 3 + jne .LBB5_4 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jb .LBB5_4 + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rsi, rax + jne .LBB5_4 + cmp word ptr [rdi], -16192 + je .LBB5_5 +.LBB5_4: + xor edi, edi + mov rdx, rsi +.LBB5_5: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..95b993c7e0a8 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca @@ -0,0 +1,81 @@ +Iterations: 100 +Instructions: 2100 +Total Cycles: 709 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.24 +IPC: 2.96 +Block RThroughput: 5.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 test dil, 3 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jb .LBB5_4 + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rsi, rax + 1 1 1.00 jne .LBB5_4 + 2 6 0.50 * cmp word ptr [rdi], -16192 + 1 1 1.00 je .LBB5_5 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.98 6.99 - 7.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.48 0.51 - 0.01 - - test dil, 3 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.51 0.49 - - - - movabs rax, 9223372036854775804 + - - 0.01 0.99 - - - - and rax, rsi + - - 0.51 0.49 - - - - cmp rax, 9 + - - - - - 1.00 - - jb .LBB5_4 + - - 0.98 - - 0.02 - - add rax, -9 + - - 0.98 0.02 - - - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.99 - - 0.01 - - shr rdx + - - - 1.00 - - - - lea rax, [rdx + 2*rdx] + - - - 0.51 - 0.49 - - or rax, 3 + - - 0.01 0.49 - 0.50 - - add rax, 9 + - - - 0.02 - 0.98 - - cmp rsi, rax + - - - - - 1.00 - - jne .LBB5_4 + - - 0.51 0.49 - - 0.50 0.50 cmp word ptr [rdi], -16192 + - - - - - 1.00 - - je .LBB5_5 + - - - - - - - - xor edi, edi + - - 0.50 0.50 - - - - mov rdx, rsi + - - 0.50 0.48 - 0.02 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs new file mode 100644 index 000000000000..fc3cfbae27a2 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..cf67afd31ce0 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_try_ref_from_bytes_dynamic_size: + mov rdx, rsi + mov rax, rdi + cmp rsi, 4 + setb cl + or cl, al + test cl, 1 + jne .LBB5_4 + lea rcx, [rdx - 4] + mov rsi, rcx + and rsi, -2 + add rsi, 4 + cmp rdx, rsi + jne .LBB5_4 + cmp word ptr [rax], -16192 + jne .LBB5_4 + shr rcx + mov rdx, rcx + ret +.LBB5_4: + xor eax, eax + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ecd7a18f6d6d --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 639 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.13 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 lea rcx, [rdx - 4] + 1 1 0.33 mov rsi, rcx + 1 1 0.33 and rsi, -2 + 1 1 0.33 add rsi, 4 + 1 1 0.33 cmp rdx, rsi + 1 1 1.00 jne .LBB5_4 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 shr rcx + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.32 6.32 - 6.36 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.33 0.66 - 0.01 - - mov rdx, rsi + - - 0.66 0.34 - - - - mov rax, rdi + - - 0.34 0.66 - - - - cmp rsi, 4 + - - 0.99 - - 0.01 - - setb cl + - - 0.01 0.99 - - - - or cl, al + - - - 1.00 - - - - test cl, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.66 0.34 - - - - lea rcx, [rdx - 4] + - - 0.33 0.66 - 0.01 - - mov rsi, rcx + - - 1.00 - - - - - and rsi, -2 + - - 0.66 0.34 - - - - add rsi, 4 + - - - 1.00 - - - - cmp rdx, rsi + - - - - - 1.00 - - jne .LBB5_4 + - - - - - 1.00 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.67 - - 0.33 - - shr rcx + - - 0.67 0.33 - - - - mov rdx, rcx + - - - - - 1.00 - - ret + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs new file mode 100644 index 000000000000..521557146324 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 new file mode 100644 index 000000000000..a11f27189e90 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 @@ -0,0 +1,13 @@ +bench_try_ref_from_bytes_static_size: + mov rax, rdi + cmp rsi, 6 + setne cl + or cl, al + test cl, 1 + jne .LBB5_2 + cmp word ptr [rax], -16192 + je .LBB5_3 +.LBB5_2: + xor eax, eax +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..e6bd20533a83 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 308 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 3.57 +IPC: 3.25 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setne cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 jne .LBB5_2 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.98 2.98 - 3.04 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.97 - 0.01 - - mov rax, rdi + - - 0.02 0.98 - - - - cmp rsi, 6 + - - 1.00 - - - - - setne cl + - - 0.97 0.02 - 0.01 - - or cl, al + - - 0.96 0.03 - 0.01 - - test cl, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - 0.01 0.98 - 0.01 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..8b9e7355e367 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3ef8d1448a50 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,21 @@ +bench_try_ref_from_bytes_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + seta cl + mov rax, rdi + test al, 3 + setne dil + or dil, cl + jne .LBB5_3 + lea rcx, [rdx + 2*rdx] + or rcx, 3 + add rcx, 9 + cmp rsi, rcx + jne .LBB5_3 + cmp word ptr [rax], -16192 + je .LBB5_4 +.LBB5_3: + xor eax, eax + mov rdx, rsi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..8131f3bd549e --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,75 @@ +Iterations: 100 +Instructions: 1800 +Total Cycles: 607 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 2.97 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 test al, 3 + 1 1 0.50 setne dil + 1 1 0.33 or dil, cl + 1 1 1.00 jne .LBB5_3 + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 or rcx, 3 + 1 1 0.33 add rcx, 9 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jne .LBB5_3 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.99 5.99 - 6.02 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - - - - 2.00 - - seta cl + - - 1.00 - - - - - mov rax, rdi + - - 0.99 0.01 - - - - test al, 3 + - - 1.00 - - - - - setne dil + - - - 0.99 - 0.01 - - or dil, cl + - - - - - 1.00 - - jne .LBB5_3 + - - 0.01 0.99 - - - - lea rcx, [rdx + 2*rdx] + - - - 1.00 - - - - or rcx, 3 + - - 0.99 0.01 - - - - add rcx, 9 + - - - 1.00 - - - - cmp rsi, rcx + - - - - - 1.00 - - jne .LBB5_3 + - - 1.00 - - - 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor eax, eax + - - 1.00 - - - - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..9ccd6fef558a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..ba34b1855bf1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 @@ -0,0 +1,18 @@ +bench_try_ref_from_bytes_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + seta cl + mov rax, rdi + or dil, cl + test dil, 1 + jne .LBB5_3 + lea rcx, [2*rdx + 4] + cmp rsi, rcx + jne .LBB5_3 + cmp word ptr [rax], -16192 + je .LBB5_4 +.LBB5_3: + xor eax, eax + mov rdx, rsi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ae049c03dfde --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,69 @@ +Iterations: 100 +Instructions: 1500 +Total Cycles: 507 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.96 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 or dil, cl + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_3 + 1 1 0.50 lea rcx, [2*rdx + 4] + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jne .LBB5_3 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.98 4.99 - 5.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.50 0.50 - - - - cmp rdx, rax + - - 1.96 - - 0.04 - - seta cl + - - 0.01 0.99 - - - - mov rax, rdi + - - 1.00 - - - - - or dil, cl + - - 0.99 0.01 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_3 + - - 0.01 0.99 - - - - lea rcx, [2*rdx + 4] + - - 0.02 0.49 - 0.49 - - cmp rsi, rcx + - - - - - 1.00 - - jne .LBB5_3 + - - - 0.51 - 0.49 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.49 0.51 - - - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs new file mode 100644 index 000000000000..23b346f9c98d --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..d832cb7ecf7f --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 @@ -0,0 +1,29 @@ +bench_try_ref_from_prefix_dynamic_padding: + xor edx, edx + mov eax, 0 + test dil, 3 + je .LBB5_1 + ret +.LBB5_1: + movabs rax, 9223372036854775804 + and rsi, rax + cmp rsi, 9 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -9 + movabs rcx, -6148914691236517205 + mov rax, rsi + mul rcx + mov rax, rdx + shr rax + movzx ecx, word ptr [rdi] + cmp cx, -16192 + mov edx, 2 + cmove rdx, rax + xor eax, eax + cmp ecx, 49344 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..482112a39b33 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca @@ -0,0 +1,91 @@ +Iterations: 100 +Instructions: 2600 +Total Cycles: 843 +Total uOps: 2900 + +Dispatch Width: 4 +uOps Per Cycle: 3.44 +IPC: 3.08 +Block RThroughput: 7.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_1 + 1 1 1.00 U ret + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rsi, rax + 1 1 0.33 cmp rsi, 9 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov rax, rsi + 2 4 1.00 mul rcx + 1 1 0.33 mov rax, rdx + 1 1 0.50 shr rax + 1 5 0.50 * movzx ecx, word ptr [rdi] + 1 1 0.33 cmp cx, -16192 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rax + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp ecx, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 8.33 8.33 - 8.34 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.32 0.34 - 0.34 - - mov eax, 0 + - - 0.34 0.33 - 0.33 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_1 + - - - - - 1.00 - - ret + - - 0.35 0.65 - - - - movabs rax, 9223372036854775804 + - - 0.96 0.03 - 0.01 - - and rsi, rax + - - 0.01 0.97 - 0.02 - - cmp rsi, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.67 0.01 - 0.32 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.02 0.34 - 0.64 - - add rsi, -9 + - - 0.33 0.66 - 0.01 - - movabs rcx, -6148914691236517205 + - - 0.66 0.34 - - - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 0.01 0.99 - - - - mov rax, rdx + - - 0.99 - - 0.01 - - shr rax + - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] + - - 0.33 0.03 - 0.64 - - cmp cx, -16192 + - - 0.01 0.31 - 0.68 - - mov edx, 2 + - - 1.00 1.00 - - - - cmove rdx, rax + - - - - - - - - xor eax, eax + - - 0.33 0.33 - 0.34 - - cmp ecx, 49344 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs new file mode 100644 index 000000000000..41a466ec8e6a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..be7f34b9f8f5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_try_ref_from_prefix_dynamic_size: + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + cmp rsi, 4 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -4 + shr rsi + movzx ecx, word ptr [rdi] + cmp ecx, 49344 + mov edx, 2 + cmove rdx, rsi + xor eax, eax + cmp cx, -16192 + cmove rax, rdi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..11706defe11e --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 573 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.66 +IPC: 3.32 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 cmp rsi, 4 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -4 + 1 1 0.50 shr rsi + 1 5 0.50 * movzx ecx, word ptr [rdi] + 1 1 0.33 cmp ecx, 49344 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rsi + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp cx, -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.66 5.67 - 5.67 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.30 0.37 - 0.33 - - mov eax, 0 + - - 0.35 0.32 - 0.33 - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.32 0.33 - 0.35 - - cmp rsi, 4 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.33 0.35 - 0.32 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.34 0.64 - 0.02 - - add rsi, -4 + - - 1.00 - - - - - shr rsi + - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] + - - 0.60 0.40 - - - - cmp ecx, 49344 + - - 0.05 0.95 - - - - mov edx, 2 + - - 1.00 1.00 - - - - cmove rdx, rsi + - - - - - - - - xor eax, eax + - - 0.37 0.31 - 0.32 - - cmp cx, -16192 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs new file mode 100644 index 000000000000..5f13d482b505 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 new file mode 100644 index 000000000000..83212f776ea6 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 @@ -0,0 +1,15 @@ +bench_try_ref_from_prefix_static_size: + cmp rsi, 6 + setb al + or al, dil + test al, 1 + jne .LBB5_2 + movzx eax, word ptr [rdi] + cmp eax, 49344 + mov eax, 2 + cmove rax, rdi + je .LBB5_3 +.LBB5_2: + xor eax, eax +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..5d02b863a741 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 374 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 3.48 +IPC: 3.21 +Block RThroughput: 3.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb al + 1 1 0.33 or al, dil + 1 1 0.33 test al, 1 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi] + 1 1 0.33 cmp eax, 49344 + 1 1 0.33 mov eax, 2 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.65 - 3.69 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.35 0.64 - 0.01 - - cmp rsi, 6 + - - 1.00 - - - - - setb al + - - 0.02 0.66 - 0.32 - - or al, dil + - - 0.03 0.65 - 0.32 - - test al, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi] + - - 0.92 0.07 - 0.01 - - cmp eax, 49344 + - - 0.37 0.63 - - - - mov eax, 2 + - - 0.97 1.00 - 0.03 - - cmove rax, rdi + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..1744a40759b1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..80e66ba1601c --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,30 @@ +bench_try_ref_from_prefix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + xor ecx, ecx + mov eax, 0 + test dil, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rax, rsi + jbe .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + movzx esi, word ptr [rdi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..512e8ce64393 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,91 @@ +Iterations: 100 +Instructions: 2600 +Total Cycles: 806 +Total uOps: 2800 + +Dispatch Width: 4 +uOps Per Cycle: 3.47 +IPC: 3.23 +Block RThroughput: 7.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rax, rsi + 1 1 1.00 jbe .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 5 0.50 * movzx esi, word ptr [rdi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.98 7.99 - 8.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.98 - - 0.02 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - - - - - - xor ecx, ecx + - - 0.99 0.01 - - - - mov eax, 0 + - - 0.01 0.96 - 0.03 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.97 0.01 - 0.02 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.03 0.97 - - - - lea rax, [rdx + 2*rdx] + - - 0.03 0.97 - - - - or rax, 3 + - - 0.01 0.99 - - - - add rax, 9 + - - - 1.00 - - - - cmp rax, rsi + - - - - - 1.00 - - jbe .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.98 0.01 - 0.01 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi] + - - 0.97 0.03 - - - - cmp si, -16192 + - - 0.98 0.01 - 0.01 - - mov ecx, 2 + - - 1.00 0.03 - 0.97 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.03 0.97 - - - - cmp esi, 49344 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - 0.03 - 0.97 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..ed0f50941194 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..c12e87c137c5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,26 @@ +bench_try_ref_from_prefix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + mov rcx, rdx + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_5 + lea rax, [2*rcx + 4] + cmp rax, rsi + jbe .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + movzx esi, word ptr [rdi] + cmp si, -16192 + mov edx, 2 + cmove rdx, rcx + xor eax, eax + cmp esi, 49344 + cmove rax, rdi +.LBB5_5: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6c3f1a1ec97a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,83 @@ +Iterations: 100 +Instructions: 2200 +Total Cycles: 674 +Total uOps: 2400 + +Dispatch Width: 4 +uOps Per Cycle: 3.56 +IPC: 3.26 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.33 mov rcx, rdx + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rcx + 4] + 1 1 0.33 cmp rax, rsi + 1 1 1.00 jbe .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 5 0.50 * movzx esi, word ptr [rdi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.65 6.66 - 6.69 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.66 0.33 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.02 0.66 - 0.32 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.66 0.33 - 0.01 - - mov rcx, rdx + - - - - - - - - xor edx, edx + - - 0.33 0.01 - 0.66 - - mov eax, 0 + - - 0.34 0.65 - 0.01 - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.65 0.35 - - - - lea rax, [2*rcx + 4] + - - - 1.00 - - - - cmp rax, rsi + - - - - - 1.00 - - jbe .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.34 0.01 - 0.65 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi] + - - 0.65 0.34 - 0.01 - - cmp si, -16192 + - - 0.66 0.34 - - - - mov edx, 2 + - - 1.00 0.99 - 0.01 - - cmove rdx, rcx + - - - - - - - - xor eax, eax + - - 0.34 0.66 - - - - cmp esi, 49344 + - - 1.00 0.99 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs new file mode 100644 index 000000000000..981feca3ca24 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..b3e924442865 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_try_ref_from_suffix_dynamic_padding: + lea eax, [rsi + rdi] + test al, 3 + jne .LBB5_1 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jae .LBB5_3 +.LBB5_1: + xor eax, eax + ret +.LBB5_3: + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rcx, [rdx + 2*rdx] + sub rsi, rcx + or rcx, -4 + add rsi, rdi + lea rdi, [rcx + rsi] + add rdi, -8 + xor eax, eax + cmp word ptr [rcx + rsi - 8], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..d56ae56d854a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 791 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 2.91 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 test al, 3 + 1 1 1.00 jne .LBB5_1 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jae .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 sub rsi, rcx + 1 1 0.33 or rcx, -4 + 1 1 0.33 add rsi, rdi + 1 1 0.50 lea rdi, [rcx + rsi] + 1 1 0.33 add rdi, -8 + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rcx + rsi - 8], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.70 7.58 - 7.72 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.26 0.74 - - - - lea eax, [rsi + rdi] + - - 0.19 0.28 - 0.53 - - test al, 3 + - - - - - 1.00 - - jne .LBB5_1 + - - 0.93 0.06 - 0.01 - - movabs rax, 9223372036854775804 + - - 0.81 0.14 - 0.05 - - and rax, rsi + - - 0.55 0.43 - 0.02 - - cmp rax, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.42 0.56 - 0.02 - - add rax, -9 + - - 0.67 0.30 - 0.03 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.71 - - 0.29 - - shr rdx + - - 0.32 0.68 - - - - lea rcx, [rdx + 2*rdx] + - - 0.57 0.04 - 0.39 - - sub rsi, rcx + - - 0.28 0.67 - 0.05 - - or rcx, -4 + - - 0.29 0.29 - 0.42 - - add rsi, rdi + - - 0.02 0.98 - - - - lea rdi, [rcx + rsi] + - - 0.02 0.41 - 0.57 - - add rdi, -8 + - - - - - - - - xor eax, eax + - - 0.57 0.01 - 0.42 0.50 0.50 cmp word ptr [rcx + rsi - 8], -16192 + - - 0.09 0.99 - 0.92 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs new file mode 100644 index 000000000000..c3d75ac9b3e1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..d51f7817e599 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 @@ -0,0 +1,18 @@ +bench_try_ref_from_suffix_dynamic_size: + lea eax, [rsi + rdi] + cmp rsi, 4 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rdx, [rsi - 4] + shr rdx + and esi, 1 + lea rcx, [rdi + rsi] + xor eax, eax + cmp word ptr [rdi + rsi], -16192 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6cf7f8e493f5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 510 +Total uOps: 1800 + +Dispatch Width: 4 +uOps Per Cycle: 3.53 +IPC: 3.14 +Block RThroughput: 4.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rdx, [rsi - 4] + 1 1 0.50 shr rdx + 1 1 0.33 and esi, 1 + 1 1 0.50 lea rcx, [rdi + rsi] + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi + rsi], -16192 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.99 5.00 - 5.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.98 0.02 - - - - lea eax, [rsi + rdi] + - - - 0.98 - 0.02 - - cmp rsi, 4 + - - 1.00 - - - - - setb cl + - - 0.01 0.99 - - - - or cl, al + - - 0.01 0.07 - 0.92 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.93 0.07 - - - - lea rdx, [rsi - 4] + - - 0.93 - - 0.07 - - shr rdx + - - 0.06 0.93 - 0.01 - - and esi, 1 + - - 0.07 0.93 - - - - lea rcx, [rdi + rsi] + - - - - - - - - xor eax, eax + - - - 0.01 - 0.99 0.50 0.50 cmp word ptr [rdi + rsi], -16192 + - - 1.00 1.00 - - - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs new file mode 100644 index 000000000000..d4b92f639a32 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 new file mode 100644 index 000000000000..cd39f70931bc --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 @@ -0,0 +1,16 @@ +bench_try_ref_from_suffix_static_size: + lea eax, [rsi + rdi] + cmp rsi, 6 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rcx, [rdi + rsi] + add rcx, -6 + xor eax, eax + cmp word ptr [rdi + rsi - 6], -16192 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..087d1e7ed971 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 443 +Total uOps: 1600 + +Dispatch Width: 4 +uOps Per Cycle: 3.61 +IPC: 3.16 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rdi + rsi] + 1 1 0.33 add rcx, -6 + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi + rsi - 6], -16192 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.68 - - - - lea eax, [rsi + rdi] + - - 0.05 0.94 - 0.01 - - cmp rsi, 6 + - - 1.00 - - - - - setb cl + - - 0.95 0.05 - - - - or cl, al + - - 0.95 0.02 - 0.03 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.04 0.96 - - - - lea rcx, [rdi + rsi] + - - 0.02 0.97 - 0.01 - - add rcx, -6 + - - - - - - - - xor eax, eax + - - 0.03 0.66 - 0.31 0.50 0.50 cmp word ptr [rdi + rsi - 6], -16192 + - - 0.97 0.05 - 0.98 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..1da455c9a238 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..c7530d8b6815 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,32 @@ +bench_try_ref_from_suffix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + lea r8, [rdi + rsi] + movzx esi, word ptr [rdi + rsi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, r8 + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..be736c00c250 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,95 @@ +Iterations: 100 +Instructions: 2800 +Total Cycles: 878 +Total uOps: 3000 + +Dispatch Width: 4 +uOps Per Cycle: 3.42 +IPC: 3.19 +Block RThroughput: 7.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.50 lea r8, [rdi + rsi] + 1 5 0.50 * movzx esi, word ptr [rdi + rsi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, r8 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 8.65 8.65 - 8.70 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.67 0.30 - 0.03 - - movabs rax, 3074457345618258598 + - - 0.01 0.99 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.99 0.01 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.35 0.62 - 0.03 - - mov eax, 0 + - - 0.99 0.01 - - - - test r8b, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.68 0.30 - 0.02 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.07 0.93 - - - - lea rax, [rdx + 2*rdx] + - - 0.06 0.35 - 0.59 - - or rax, 3 + - - 0.02 0.07 - 0.91 - - add rax, 9 + - - 0.01 0.04 - 0.95 - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.92 0.01 - 0.07 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - 1.00 - - - - lea r8, [rdi + rsi] + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi + rsi] + - - 0.01 0.99 - - - - cmp si, -16192 + - - 0.88 0.04 - 0.08 - - mov ecx, 2 + - - 1.00 0.99 - 0.01 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.99 0.01 - - - - cmp esi, 49344 + - - 1.00 1.00 - - - - cmove rax, r8 + - - - 0.99 - 0.01 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..8c2b80f8762f --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..952eb12de8d6 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,28 @@ +bench_try_ref_from_suffix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 1 + jne .LBB5_5 + lea rax, [2*rdx + 4] + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + lea r8, [rdi + rsi] + movzx esi, word ptr [rdi + rsi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, r8 +.LBB5_5: + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..d4f78f67a252 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 1107 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 2.35 +IPC: 2.17 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rdx + 4] + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.50 lea r8, [rdi + rsi] + 1 5 0.50 * movzx esi, word ptr [rdi + rsi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, r8 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 7.00 - 8.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.95 - 0.03 - - movabs rax, 4611686018427387901 + - - 0.93 0.04 - 0.03 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.96 0.04 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.95 0.02 - 0.03 - - mov eax, 0 + - - 0.95 0.05 - - - - test r8b, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.06 0.94 - - - - lea rax, [2*rdx + 4] + - - 0.93 0.07 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.03 0.95 - 0.02 - - mov edx, 1 + - - - - - 1.00 - - ret + - - 0.97 0.03 - - - - lea r8, [rdi + rsi] + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi + rsi] + - - 0.03 0.97 - - - - cmp si, -16192 + - - 0.05 0.94 - 0.01 - - mov ecx, 2 + - - 0.06 0.98 - 0.96 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.97 0.03 - - - - cmp esi, 49344 + - - 0.06 0.96 - 0.98 - - cmove rax, r8 + - - 0.02 0.03 - 0.95 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute.rs b/rust/zerocopy/benches/try_transmute.rs new file mode 100644 index 000000000000..c0de07a8d094 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.rs @@ -0,0 +1,16 @@ +use zerocopy::Unalign; +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C)] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute(source: MinimalViableSource) -> Option> { + zerocopy::try_transmute!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute.x86-64 b/rust/zerocopy/benches/try_transmute.x86-64 new file mode 100644 index 000000000000..9e16a663257c --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.x86-64 @@ -0,0 +1,9 @@ +bench_try_transmute: + movzx ecx, di + xor eax, eax + cmp ecx, 49344 + sete al + and rdi, -65536 + xor rax, 49345 + or rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute.x86-64.mca b/rust/zerocopy/benches/try_transmute.x86-64.mca new file mode 100644 index 000000000000..33abc3bf341e --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.x86-64.mca @@ -0,0 +1,55 @@ +Iterations: 100 +Instructions: 800 +Total Cycles: 238 +Total uOps: 800 + +Dispatch Width: 4 +uOps Per Cycle: 3.36 +IPC: 3.36 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movzx ecx, di + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp ecx, 49344 + 1 1 0.50 sete al + 1 1 0.33 and rdi, -65536 + 1 1 0.33 xor rax, 49345 + 1 1 0.33 or rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.33 2.33 - 2.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.67 - 0.01 - - movzx ecx, di + - - - - - - - - xor eax, eax + - - 0.33 0.67 - - - - cmp ecx, 49344 + - - 1.00 - - - - - sete al + - - 0.67 0.33 - - - - and rdi, -65536 + - - - 0.66 - 0.34 - - xor rax, 49345 + - - 0.01 - - 0.99 - - or rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs new file mode 100644 index 000000000000..c9236e13d23c --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs @@ -0,0 +1,18 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + header: [u8; 6], + trailer: [[u8; 2]], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute_ref_dynamic_size( + source: &MinimalViableSource, +) -> Option<&format::CocoPacket> { + zerocopy::try_transmute_ref!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 new file mode 100644 index 000000000000..d34d2b3eb3df --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 @@ -0,0 +1,6 @@ +bench_try_transmute_ref_dynamic_size: + lea rdx, [rsi + 1] + xor eax, eax + cmp word ptr [rdi], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..bc771b504713 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 209 +Total uOps: 700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.39 +Block RThroughput: 1.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rdx, [rsi + 1] + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.51 - 1.99 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.51 0.49 - - - - lea rdx, [rsi + 1] + - - - - - - - - xor eax, eax + - - - 0.02 - 0.98 0.50 0.50 cmp word ptr [rdi], -16192 + - - 0.99 1.00 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.rs b/rust/zerocopy/benches/try_transmute_ref_static_size.rs new file mode 100644 index 000000000000..631cce2b0bb9 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.rs @@ -0,0 +1,17 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute_ref_static_size( + source: &MinimalViableSource, +) -> Option<&format::CocoPacket> { + zerocopy::try_transmute_ref!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 new file mode 100644 index 000000000000..8c16face9875 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 @@ -0,0 +1,5 @@ +bench_try_transmute_ref_static_size: + xor eax, eax + cmp word ptr [rdi], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca new file mode 100644 index 000000000000..cf7384989536 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 160 +Total uOps: 600 + +Dispatch Width: 4 +uOps Per Cycle: 3.75 +IPC: 2.50 +Block RThroughput: 1.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.02 1.48 - 1.50 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - 0.02 0.49 - 0.49 0.50 0.50 cmp word ptr [rdi], -16192 + - - 1.00 0.99 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_dynamic_size.rs b/rust/zerocopy/benches/write_to_dynamic_size.rs new file mode 100644 index 000000000000..c126a1468c9b --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_dynamic_size(source: &format::CocoPacket, destination: &mut [u8]) -> Option<()> { + source.write_to(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_dynamic_size.x86-64 new file mode 100644 index 000000000000..c5abb17f7eba --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.x86-64 @@ -0,0 +1,21 @@ +bench_write_to_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + cmp rcx, r14 + jne .LBB5_2 + mov rax, rdi + mov rdi, rdx + mov rsi, rax + mov rdx, rbx + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp rbx, r14 + sete al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..5b2c08a31a29 --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2890 +Total uOps: 2500 + +Dispatch Width: 4 +uOps Per Cycle: 0.87 +IPC: 0.66 +Block RThroughput: 6.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 cmp rcx, r14 + 1 1 1.00 jne .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, rbx + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp rbx, r14 + 1 1 0.50 sete al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.66 4.64 4.00 4.70 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.02 0.97 - 0.01 - - mov rbx, rcx + - - 0.97 0.03 - - - - lea r14, [2*rsi + 5] + - - 0.63 0.35 - 0.02 - - and r14, -2 + - - 0.31 0.34 - 0.35 - - cmp rcx, r14 + - - - - - 1.00 - - jne .LBB5_2 + - - 0.33 0.33 - 0.34 - - mov rax, rdi + - - 0.36 0.31 - 0.33 - - mov rdi, rdx + - - 0.33 0.35 - 0.32 - - mov rsi, rax + - - 0.35 0.63 - 0.02 - - mov rdx, rbx + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.65 0.35 - - - - cmp rbx, r14 + - - 0.69 - - 0.31 - - sete al + - - 0.02 0.98 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs b/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs new file mode 100644 index 000000000000..a54d32773113 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_prefix_dynamic_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_prefix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..d7779c6c9178 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 @@ -0,0 +1,21 @@ +bench_write_to_prefix_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + cmp r14, rcx + ja .LBB5_2 + mov rax, rdi + mov rdi, rdx + mov rsi, rax + mov rdx, r14 + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp r14, rbx + setbe al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4cebe24d4f3b --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2890 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 0.90 +IPC: 0.66 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 cmp r14, rcx + 1 1 1.00 ja .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, r14 + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp r14, rbx + 2 2 1.00 setbe al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.47 4.49 4.00 5.04 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.48 0.51 - 0.01 - - mov rbx, rcx + - - 0.51 0.49 - - - - lea r14, [2*rsi + 5] + - - 0.48 0.05 - 0.47 - - and r14, -2 + - - 0.48 0.49 - 0.03 - - cmp r14, rcx + - - - - - 1.00 - - ja .LBB5_2 + - - 0.04 0.47 - 0.49 - - mov rax, rdi + - - 0.49 0.03 - 0.48 - - mov rdi, rdx + - - 0.03 0.48 - 0.49 - - mov rsi, rax + - - 0.48 0.51 - 0.01 - - mov rdx, r14 + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.51 0.49 - - - - cmp r14, rbx + - - 1.94 - - 0.06 - - setbe al + - - 0.03 0.97 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.rs b/rust/zerocopy/benches/write_to_prefix_static_size.rs new file mode 100644 index 000000000000..826222c129fe --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_prefix_static_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_prefix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 new file mode 100644 index 000000000000..9cf066295304 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_prefix_static_size: + cmp rdx, 6 + jb .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + 4], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi], eax +.LBB5_2: + cmp rdx, 6 + setae al + ret diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..5d17200abd2d --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + 4], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setae al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + 4], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - setae al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_static_size.rs b/rust/zerocopy/benches/write_to_static_size.rs new file mode 100644 index 000000000000..3bb9435c5ade --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_static_size(source: &format::CocoPacket, destination: &mut [u8]) -> Option<()> { + source.write_to(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_static_size.x86-64 b/rust/zerocopy/benches/write_to_static_size.x86-64 new file mode 100644 index 000000000000..d6413e0fd614 --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_static_size: + cmp rdx, 6 + jne .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + 4], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi], eax +.LBB5_2: + cmp rdx, 6 + sete al + ret diff --git a/rust/zerocopy/benches/write_to_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_static_size.x86-64.mca new file mode 100644 index 000000000000..cc5bb1d26fc0 --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + 4], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 sete al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + 4], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - sete al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs b/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs new file mode 100644 index 000000000000..9fa6b91cda41 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_suffix_dynamic_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_suffix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..75f349562db6 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_write_to_suffix_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + sub rcx, r14 + jb .LBB5_2 + mov rax, rdi + add rdx, rcx + mov rdi, rdx + mov rsi, rax + mov rdx, r14 + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp rbx, r14 + setae al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..95cb9dfe2eb7 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 2890 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 0.90 +IPC: 0.69 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 sub rcx, r14 + 1 1 1.00 jb .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 add rdx, rcx + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, r14 + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp rbx, r14 + 1 1 0.50 setae al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.98 4.98 4.00 5.04 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.94 0.05 - 0.01 - - mov rbx, rcx + - - 0.06 0.94 - - - - lea r14, [2*rsi + 5] + - - 0.93 0.02 - 0.05 - - and r14, -2 + - - 0.05 0.94 - 0.01 - - sub rcx, r14 + - - - - - 1.00 - - jb .LBB5_2 + - - 0.02 0.04 - 0.94 - - mov rax, rdi + - - 0.03 0.97 - - - - add rdx, rcx + - - 0.95 0.05 - - - - mov rdi, rdx + - - 0.94 0.03 - 0.03 - - mov rsi, rax + - - 0.01 0.03 - 0.96 - - mov rdx, r14 + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.05 0.94 - 0.01 - - cmp rbx, r14 + - - 0.97 - - 0.03 - - setae al + - - 0.03 0.97 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.rs b/rust/zerocopy/benches/write_to_suffix_static_size.rs new file mode 100644 index 000000000000..1c95aba4b16c --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_suffix_static_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_suffix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 new file mode 100644 index 000000000000..934aa370d4d6 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_suffix_static_size: + cmp rdx, 6 + jb .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + rdx - 2], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi + rdx - 6], eax +.LBB5_2: + cmp rdx, 6 + setae al + ret diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..6b18e4a44585 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + rdx - 2], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi + rdx - 6], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setae al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + rdx - 2], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi + rdx - 6], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - setae al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/zero_dynamic_padding.rs b/rust/zerocopy/benches/zero_dynamic_padding.rs new file mode 100644 index 000000000000..8eda0953d1e6 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_dynamic_padding(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_dynamic_padding.x86-64 b/rust/zerocopy/benches/zero_dynamic_padding.x86-64 new file mode 100644 index 000000000000..7dccf1745f81 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.x86-64 @@ -0,0 +1,7 @@ +bench_zero_dynamic_padding: + lea rax, [rsi + 2*rsi] + movabs rdx, 9223372036854775804 + and rdx, rax + add rdx, 12 + xor esi, esi + jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..098fc107875f --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca @@ -0,0 +1,51 @@ +Iterations: 100 +Instructions: 600 +Total Cycles: 209 +Total uOps: 700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.87 +Block RThroughput: 1.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rax, [rsi + 2*rsi] + 1 1 0.33 movabs rdx, 9223372036854775804 + 1 1 0.33 and rdx, rax + 1 1 0.33 add rdx, 12 + 1 0 0.25 xor esi, esi + 2 6 1.00 * jmp qword ptr [rip + memset@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 - 1.68 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.33 0.67 - - - - lea rax, [rsi + 2*rsi] + - - 0.98 - - 0.02 - - movabs rdx, 9223372036854775804 + - - 0.01 0.66 - 0.33 - - and rdx, rax + - - 0.34 0.33 - 0.33 - - add rdx, 12 + - - - - - - - - xor esi, esi + - - - - - 1.00 0.50 0.50 jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_size.rs b/rust/zerocopy/benches/zero_dynamic_size.rs new file mode 100644 index 000000000000..536d800ebc76 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_dynamic_size(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_dynamic_size.x86-64 b/rust/zerocopy/benches/zero_dynamic_size.x86-64 new file mode 100644 index 000000000000..2b31ed644eaf --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.x86-64 @@ -0,0 +1,5 @@ +bench_zero_dynamic_size: + lea rdx, [2*rsi + 5] + and rdx, -2 + xor esi, esi + jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca b/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..0b086a29b0ed --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 142 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 3.52 +IPC: 2.82 +Block RThroughput: 1.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rdx, [2*rsi + 5] + 1 1 0.33 and rdx, -2 + 1 0 0.25 xor esi, esi + 2 6 1.00 * jmp qword ptr [rip + memset@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 0.01 - - - - lea rdx, [2*rsi + 5] + - - - 0.99 - 0.01 - - and rdx, -2 + - - - - - - - - xor esi, esi + - - - - - 1.00 0.50 0.50 jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_static_size.rs b/rust/zerocopy/benches/zero_static_size.rs new file mode 100644 index 000000000000..fa7fa0839c15 --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_static_size(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_static_size.x86-64 b/rust/zerocopy/benches/zero_static_size.x86-64 new file mode 100644 index 000000000000..ced8e18949b2 --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.x86-64 @@ -0,0 +1,4 @@ +bench_zero_static_size: + mov word ptr [rdi + 4], 0 + mov dword ptr [rdi], 0 + ret diff --git a/rust/zerocopy/benches/zero_static_size.x86-64.mca b/rust/zerocopy/benches/zero_static_size.x86-64.mca new file mode 100644 index 000000000000..042897ef2dfe --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 203 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 1.48 +IPC: 1.48 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 1.00 * mov word ptr [rdi + 4], 0 + 1 1 1.00 * mov dword ptr [rdi], 0 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - 2.00 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 mov word ptr [rdi + 4], 0 + - - - - 1.00 - 1.00 - mov dword ptr [rdi], 0 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/rustdoc/style.css b/rust/zerocopy/rustdoc/style.css new file mode 100644 index 000000000000..c5c8aeb743a1 --- /dev/null +++ b/rust/zerocopy/rustdoc/style.css @@ -0,0 +1,55 @@ +/* +Copyright 2026 The Fuchsia Authors + +Licensed under a BSD-style license , Apache License, Version 2.0 +, or the MIT +license , at your option. +This file may not be copied, modified, or distributed except according to +those terms. +*/ + +.codegen-tabs { + display: grid; + grid-template-columns: repeat(var(--arity), minmax(200px, 1fr)); + grid-template-rows: auto 1fr; + column-gap: 1rem; +} + +.codegen-tabs:not(:has(> details[open]))::after { + grid-column: 1/-1; + content: 'Click one of the above headers to expand its contents.'; + font-style: italic; + font-size: small; + text-align: center; +} + +.codegen-tabs details { + display: grid; + grid-column: 1 / -1; + grid-row: 1 / span 2; + grid-template-columns: subgrid; + grid-template-rows: subgrid; +} + +.codegen-tabs summary { + display: grid; + grid-column: var(--n) / span 1; + grid-row: 1; + z-index: 1; + border-bottom: 2px solid var(--headings-border-bottom-color); + cursor: pointer; +} + +.codegen-tabs details[open] > summary { + background-color: var(--code-block-background-color); + border-bottom-color: var(--target-border-color); +} + +.codegen-tabs details::details-content { + grid-column: 1 / -1; + grid-row: 2; +} + +.codegen-tabs details:not([open])::details-content { + display: none; +} diff --git a/rust/zerocopy/src/byte_slice.rs b/rust/zerocopy/src/byte_slice.rs new file mode 100644 index 000000000000..6f9ee9ac3336 --- /dev/null +++ b/rust/zerocopy/src/byte_slice.rs @@ -0,0 +1,432 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Traits for types that encapsulate a `[u8]`. +//! +//! These traits are used to bound the `B` parameter of [`Ref`]. + +use core::{ + cell, + ops::{Deref, DerefMut}, +}; + +// For each trait polyfill, as soon as the corresponding feature is stable, the +// polyfill import will be unused because method/function resolution will prefer +// the inherent method/function over a trait method/function. Thus, we suppress +// the `unused_imports` warning. +// +// See the documentation on `util::polyfills` for more information. +#[allow(unused_imports)] +use crate::util::polyfills::{self, NonNullExt as _, NumExt as _}; +#[cfg(doc)] +use crate::Ref; + +/// A mutable or immutable reference to a byte slice. +/// +/// `ByteSlice` abstracts over the mutability of a byte slice reference, and is +/// implemented for various special reference types such as +/// [`Ref<[u8]>`](core::cell::Ref) and [`RefMut<[u8]>`](core::cell::RefMut). +/// +/// # Safety +/// +/// Implementations of `ByteSlice` must promise that their implementations of +/// [`Deref`] and [`DerefMut`] are "stable". In particular, given `B: ByteSlice` +/// and `b: B`, two calls, each to either `b.deref()` or `b.deref_mut()`, must +/// return a byte slice with the same address and length. This must hold even if +/// the two calls are separated by an arbitrary sequence of calls to methods on +/// `ByteSlice`, [`ByteSliceMut`], [`IntoByteSlice`], or [`IntoByteSliceMut`], +/// or on their super-traits. This does *not* need to hold if the two calls are +/// separated by any method calls, field accesses, or field modifications *other +/// than* those from these traits. +/// +/// Note that this also implies that, given `b: B`, the address and length +/// cannot be modified via objects other than `b`, either on the same thread or +/// on another thread. +pub unsafe trait ByteSlice: Deref + Sized {} + +/// A mutable reference to a byte slice. +/// +/// `ByteSliceMut` abstracts over various ways of storing a mutable reference to +/// a byte slice, and is implemented for various special reference types such as +/// `RefMut<[u8]>`. +/// +/// `ByteSliceMut` is a shorthand for [`ByteSlice`] and [`DerefMut`]. +pub trait ByteSliceMut: ByteSlice + DerefMut {} +impl ByteSliceMut for B {} + +/// A [`ByteSlice`] which can be copied without violating dereference stability. +/// +/// # Safety +/// +/// If `B: CopyableByteSlice`, then the dereference stability properties +/// required by [`ByteSlice`] (see that trait's safety documentation) do not +/// only hold regarding two calls to `b.deref()` or `b.deref_mut()`, but also +/// hold regarding `c.deref()` or `c.deref_mut()`, where `c` is produced by +/// copying `b`. +pub unsafe trait CopyableByteSlice: ByteSlice + Copy + CloneableByteSlice {} + +/// A [`ByteSlice`] which can be cloned without violating dereference stability. +/// +/// # Safety +/// +/// If `B: CloneableByteSlice`, then the dereference stability properties +/// required by [`ByteSlice`] (see that trait's safety documentation) do not +/// only hold regarding two calls to `b.deref()` or `b.deref_mut()`, but also +/// hold regarding `c.deref()` or `c.deref_mut()`, where `c` is produced by +/// `b.clone()`, `b.clone().clone()`, etc. +pub unsafe trait CloneableByteSlice: ByteSlice + Clone {} + +/// A [`ByteSlice`] that can be split in two. +/// +/// # Safety +/// +/// Unsafe code may depend for its soundness on the assumption that `split_at` +/// and `split_at_unchecked` are implemented correctly. In particular, given `B: +/// SplitByteSlice` and `b: B`, if `b.deref()` returns a byte slice with address +/// `addr` and length `len`, then if `split <= len`, both of these +/// invocations: +/// - `b.split_at(split)` +/// - `b.split_at_unchecked(split)` +/// +/// ...will return `(first, second)` such that: +/// - `first`'s address is `addr` and its length is `split` +/// - `second`'s address is `addr + split` and its length is `len - split` +pub unsafe trait SplitByteSlice: ByteSlice { + /// Attempts to split `self` at the midpoint. + /// + /// `s.split_at(mid)` returns `Ok((s[..mid], s[mid..]))` if `mid <= + /// s.deref().len()` and otherwise returns `Err(s)`. + /// + /// # Safety + /// + /// Unsafe code may rely on this function correctly implementing the above + /// functionality. + #[inline] + fn split_at(self, mid: usize) -> Result<(Self, Self), Self> { + if mid <= self.deref().len() { + // SAFETY: Above, we ensure that `mid <= self.deref().len()`. By + // invariant on `ByteSlice`, a supertrait of `SplitByteSlice`, + // `.deref()` is guaranteed to be "stable"; i.e., it will always + // dereference to a byte slice of the same address and length. Thus, + // we can be sure that the above precondition remains satisfied + // through the call to `split_at_unchecked`. + unsafe { Ok(self.split_at_unchecked(mid)) } + } else { + Err(self) + } + } + + /// Splits the slice at the midpoint, possibly omitting bounds checks. + /// + /// `s.split_at_unchecked(mid)` returns `s[..mid]` and `s[mid..]`. + /// + /// # Safety + /// + /// `mid` must not be greater than `self.deref().len()`. + /// + /// # Panics + /// + /// Implementations of this method may choose to perform a bounds check and + /// panic if `mid > self.deref().len()`. They may also panic for any other + /// reason. Since it is optional, callers must not rely on this behavior for + /// soundness. + #[must_use] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self); +} + +/// A shorthand for [`SplitByteSlice`] and [`ByteSliceMut`]. +pub trait SplitByteSliceMut: SplitByteSlice + ByteSliceMut {} +impl SplitByteSliceMut for B {} + +#[allow(clippy::missing_safety_doc)] // There's a `Safety` section on `into_byte_slice`. +/// A [`ByteSlice`] that conveys no ownership, and so can be converted into a +/// byte slice. +/// +/// Some `ByteSlice` types (notably, the standard library's [`Ref`] type) convey +/// ownership, and so they cannot soundly be moved by-value into a byte slice +/// type (`&[u8]`). Some methods in this crate's API (such as [`Ref::into_ref`]) +/// are only compatible with `ByteSlice` types without these ownership +/// semantics. +/// +/// [`Ref`]: core::cell::Ref +pub unsafe trait IntoByteSlice<'a>: ByteSlice { + /// Coverts `self` into a `&[u8]`. + /// + /// # Safety + /// + /// The returned reference has the same address and length as `self.deref()` + /// and `self.deref_mut()`. + /// + /// Note that, combined with the safety invariant on [`ByteSlice`], this + /// safety invariant implies that the returned reference is "stable" in the + /// sense described in the `ByteSlice` docs. + fn into_byte_slice(self) -> &'a [u8]; +} + +#[allow(clippy::missing_safety_doc)] // There's a `Safety` section on `into_byte_slice_mut`. +/// A [`ByteSliceMut`] that conveys no ownership, and so can be converted into a +/// mutable byte slice. +/// +/// Some `ByteSliceMut` types (notably, the standard library's [`RefMut`] type) +/// convey ownership, and so they cannot soundly be moved by-value into a byte +/// slice type (`&mut [u8]`). Some methods in this crate's API (such as +/// [`Ref::into_mut`]) are only compatible with `ByteSliceMut` types without +/// these ownership semantics. +/// +/// [`RefMut`]: core::cell::RefMut +pub unsafe trait IntoByteSliceMut<'a>: IntoByteSlice<'a> + ByteSliceMut { + /// Coverts `self` into a `&mut [u8]`. + /// + /// # Safety + /// + /// The returned reference has the same address and length as `self.deref()` + /// and `self.deref_mut()`. + /// + /// Note that, combined with the safety invariant on [`ByteSlice`], this + /// safety invariant implies that the returned reference is "stable" in the + /// sense described in the `ByteSlice` docs. + fn into_byte_slice_mut(self) -> &'a mut [u8]; +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for &[u8] {} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl CopyableByteSlice for &[u8] {} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl CloneableByteSlice for &[u8] {} + +// SAFETY: This delegates to `polyfills:split_at_unchecked`, which is documented +// to correctly split `self` into two slices at the given `mid` point. +unsafe impl SplitByteSlice for &[u8] { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + (<[u8]>::get_unchecked(self, ..mid), <[u8]>::get_unchecked(self, mid..)) + } + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSlice<'a> for &'a [u8] { + #[inline(always)] + fn into_byte_slice(self) -> &'a [u8] { + // SAFETY: It would be patently insane to implement `::deref` as anything other than `fn deref(&self) -> &[u8] { + // *self }`. Assuming this holds, then `self` is stable as required by + // `into_byte_slice`. + self + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for &mut [u8] {} + +// SAFETY: This delegates to `polyfills:split_at_mut_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for &mut [u8] { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + use core::slice::from_raw_parts_mut; + + // `l_ptr` is non-null, because `self` is non-null, by invariant on + // `&mut [u8]`. + let l_ptr = self.as_mut_ptr(); + + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + let r_ptr = unsafe { l_ptr.add(mid) }; + + let l_len = mid; + + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + // + // FIXME(#67): Remove this allow. See NumExt for more details. + #[allow(unstable_name_collisions)] + let r_len = unsafe { self.len().unchecked_sub(mid) }; + + // SAFETY: These invocations of `from_raw_parts_mut` satisfy its + // documented safety preconditions [1]: + // - The data `l_ptr` and `r_ptr` are valid for both reads and writes of + // `l_len` and `r_len` bytes, respectively, and they are trivially + // aligned. In particular: + // - The entire memory range of each slice is contained within a + // single allocated object, since `l_ptr` and `r_ptr` are both + // derived from within the address range of `self`. + // - Both `l_ptr` and `r_ptr` are non-null and trivially aligned. + // `self` is non-null by invariant on `&mut [u8]`, and the + // operations that derive `l_ptr` and `r_ptr` from `self` do not + // nullify either pointer. + // - The data `l_ptr` and `r_ptr` point to `l_len` and `r_len`, + // respectively, consecutive properly initialized values of type `u8`. + // This is true for `self` by invariant on `&mut [u8]`, and remains + // true for these two sub-slices of `self`. + // - The memory referenced by the returned slice cannot be accessed + // through any other pointer (not derived from the return value) for + // the duration of lifetime `'a``, because: + // - `split_at_unchecked` consumes `self` (which is not `Copy`), + // - `split_at_unchecked` does not exfiltrate any references to this + // memory, besides those references returned below, + // - the returned slices are non-overlapping. + // - The individual sizes of the sub-slices of `self` are no larger than + // `isize::MAX`, because their combined sizes are no larger than + // `isize::MAX`, by invariant on `self`. + // + // [1] https://doc.rust-lang.org/std/slice/fn.from_raw_parts_mut.html#safety + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + (from_raw_parts_mut(l_ptr, l_len), from_raw_parts_mut(r_ptr, r_len)) + } + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSlice<'a> for &'a mut [u8] { + #[inline(always)] + fn into_byte_slice(self) -> &'a [u8] { + // SAFETY: It would be patently insane to implement `::deref` as anything other than `fn deref(&self) -> &[u8] { + // *self }`. Assuming this holds, then `self` is stable as required by + // `into_byte_slice`. + self + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSliceMut<'a> for &'a mut [u8] { + #[inline(always)] + fn into_byte_slice_mut(self) -> &'a mut [u8] { + // SAFETY: It would be patently insane to implement `::deref` as anything other than `fn deref_mut(&mut self) -> &mut + // [u8] { *self }`. Assuming this holds, then `self` is stable as + // required by `into_byte_slice_mut`. + self + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for cell::Ref<'_, [u8]> {} + +// SAFETY: This delegates to stdlib implementation of `Ref::map_split`, which is +// assumed to be correct, and `SplitByteSlice::split_at_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for cell::Ref<'_, [u8]> { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + cell::Ref::map_split(self, |slice| + // SAFETY: By precondition on caller, `mid` is not greater than + // `slice.len()`. + unsafe { + SplitByteSlice::split_at_unchecked(slice, mid) + }) + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for cell::RefMut<'_, [u8]> {} + +// SAFETY: This delegates to stdlib implementation of `RefMut::map_split`, which +// is assumed to be correct, and `SplitByteSlice::split_at_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for cell::RefMut<'_, [u8]> { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + cell::RefMut::map_split(self, |slice| + // SAFETY: By precondition on caller, `mid` is not greater than + // `slice.len()` + unsafe { + SplitByteSlice::split_at_unchecked(slice, mid) + }) + } +} + +#[cfg(kani)] +mod proofs { + use super::*; + + fn any_vec() -> Vec { + let len = kani::any(); + kani::assume(len <= crate::DstLayout::MAX_SIZE); + vec![0u8; len] + } + + #[kani::proof] + fn prove_split_at_unchecked() { + let v = any_vec(); + let slc = v.as_slice(); + let mid = kani::any(); + kani::assume(mid <= slc.len()); + let (l, r) = unsafe { slc.split_at_unchecked(mid) }; + assert_eq!(l.len() + r.len(), slc.len()); + + let slc: *const _ = slc; + let l: *const _ = l; + let r: *const _ = r; + + assert_eq!(slc.cast::(), l.cast::()); + assert_eq!(unsafe { slc.cast::().add(mid) }, r.cast::()); + + let mut v = any_vec(); + let slc = v.as_mut_slice(); + let len = slc.len(); + let mid = kani::any(); + kani::assume(mid <= slc.len()); + let (l, r) = unsafe { slc.split_at_unchecked(mid) }; + assert_eq!(l.len() + r.len(), len); + + let l: *mut _ = l; + let r: *mut _ = r; + let slc: *mut _ = slc; + + assert_eq!(slc.cast::(), l.cast::()); + assert_eq!(unsafe { slc.cast::().add(mid) }, r.cast::()); + } +} + +#[cfg(test)] +mod tests { + use core::cell::RefCell; + + use super::*; + + #[test] + fn test_ref_split_at_unchecked() { + let cell = RefCell::new([1, 2, 3, 4]); + let borrow = cell.borrow(); + let slice_ref: cell::Ref<'_, [u8]> = cell::Ref::map(borrow, |a| &a[..]); + // SAFETY: 2 is within bounds of [1, 2, 3, 4] + let (l, r) = unsafe { slice_ref.split_at_unchecked(2) }; + assert_eq!(*l, [1, 2]); + assert_eq!(*r, [3, 4]); + } + + #[test] + fn test_ref_mut_split_at_unchecked() { + let cell = RefCell::new([1, 2, 3, 4]); + let borrow_mut = cell.borrow_mut(); + let slice_ref_mut: cell::RefMut<'_, [u8]> = cell::RefMut::map(borrow_mut, |a| &mut a[..]); + // SAFETY: 2 is within bounds of [1, 2, 3, 4] + let (l, r) = unsafe { slice_ref_mut.split_at_unchecked(2) }; + assert_eq!(*l, [1, 2]); + assert_eq!(*r, [3, 4]); + } +} diff --git a/rust/zerocopy/src/byteorder.rs b/rust/zerocopy/src/byteorder.rs new file mode 100644 index 000000000000..257505203415 --- /dev/null +++ b/rust/zerocopy/src/byteorder.rs @@ -0,0 +1,1563 @@ +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Byte order-aware numeric primitives. +//! +//! This module contains equivalents of the native multi-byte integer types with +//! no alignment requirement and supporting byte order conversions. +//! +//! For each native multi-byte integer type - `u16`, `i16`, `u32`, etc - and +//! floating point type - `f32` and `f64` - an equivalent type is defined by +//! this module - [`U16`], [`I16`], [`U32`], [`F32`], [`F64`], etc. Unlike their +//! native counterparts, these types have alignment 1, and take a type parameter +//! specifying the byte order in which the bytes are stored in memory. Each type +//! implements this crate's relevant conversion and marker traits. +//! +//! These two properties, taken together, make these types useful for defining +//! data structures whose memory layout matches a wire format such as that of a +//! network protocol or a file format. Such formats often have multi-byte values +//! at offsets that do not respect the alignment requirements of the equivalent +//! native types, and stored in a byte order not necessarily the same as that of +//! the target platform. +//! +//! Type aliases are provided for common byte orders in the [`big_endian`], +//! [`little_endian`], [`network_endian`], and [`native_endian`] submodules. +//! Note that network-endian is a synonym for big-endian. +//! +//! # Example +//! +//! One use of these types is for representing network packet formats, such as +//! UDP: +//! +//! ```rust +//! use zerocopy::{*, byteorder::network_endian::U16}; +//! # use zerocopy_derive::*; +//! +//! #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] +//! #[repr(C)] +//! struct UdpHeader { +//! src_port: U16, +//! dst_port: U16, +//! length: U16, +//! checksum: U16, +//! } +//! +//! #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] +//! #[repr(C, packed)] +//! struct UdpPacket { +//! header: UdpHeader, +//! body: [u8], +//! } +//! +//! impl UdpPacket { +//! fn parse(bytes: &[u8]) -> Option<&UdpPacket> { +//! UdpPacket::ref_from_bytes(bytes).ok() +//! } +//! } +//! ``` + +use core::{ + convert::{TryFrom, TryInto}, + fmt::{Binary, Debug, LowerHex, Octal, UpperHex}, + hash::Hash, + num::TryFromIntError, +}; + +use super::*; + +/// A type-level representation of byte order. +/// +/// This type is implemented by [`BigEndian`] and [`LittleEndian`], which +/// represent big-endian and little-endian byte order respectively. This module +/// also provides a number of useful aliases for those types: [`NativeEndian`], +/// [`NetworkEndian`], [`BE`], and [`LE`]. +/// +/// `ByteOrder` types can be used to specify the byte order of the types in this +/// module - for example, [`U32`] is a 32-bit integer stored in +/// big-endian byte order. +/// +/// [`U32`]: U32 +pub trait ByteOrder: + Copy + Clone + Debug + Display + Eq + PartialEq + Ord + PartialOrd + Hash + private::Sealed +{ + #[doc(hidden)] + const ORDER: Order; +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::BigEndian {} + impl Sealed for super::LittleEndian {} +} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +#[doc(hidden)] +pub enum Order { + BigEndian, + LittleEndian, +} + +/// Big-endian byte order. +/// +/// See [`ByteOrder`] for more details. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum BigEndian {} + +impl ByteOrder for BigEndian { + const ORDER: Order = Order::BigEndian; +} + +impl Display for BigEndian { + #[inline] + fn fmt(&self, _: &mut Formatter<'_>) -> fmt::Result { + match *self {} + } +} + +/// Little-endian byte order. +/// +/// See [`ByteOrder`] for more details. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum LittleEndian {} + +impl ByteOrder for LittleEndian { + const ORDER: Order = Order::LittleEndian; +} + +impl Display for LittleEndian { + #[inline] + fn fmt(&self, _: &mut Formatter<'_>) -> fmt::Result { + match *self {} + } +} + +/// The endianness used by this platform. +/// +/// This is a type alias for [`BigEndian`] or [`LittleEndian`] depending on the +/// endianness of the target platform. +#[cfg(target_endian = "big")] +pub type NativeEndian = BigEndian; + +/// The endianness used by this platform. +/// +/// This is a type alias for [`BigEndian`] or [`LittleEndian`] depending on the +/// endianness of the target platform. +#[cfg(target_endian = "little")] +pub type NativeEndian = LittleEndian; + +/// The endianness used in many network protocols. +/// +/// This is a type alias for [`BigEndian`]. +pub type NetworkEndian = BigEndian; + +/// A type alias for [`BigEndian`]. +pub type BE = BigEndian; + +/// A type alias for [`LittleEndian`]. +pub type LE = LittleEndian; + +macro_rules! impl_fmt_trait { + ($name:ident, $native:ident, $trait:ident) => { + impl $trait for $name { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + $trait::fmt(&self.get(), f) + } + } + }; +} + +macro_rules! impl_fmt_traits { + ($name:ident, $native:ident, "floating point number") => { + impl_fmt_trait!($name, $native, Display); + }; + ($name:ident, $native:ident, "unsigned integer") => { + impl_fmt_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, "signed integer") => { + impl_fmt_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, @all_types) => { + impl_fmt_trait!($name, $native, Display); + impl_fmt_trait!($name, $native, Octal); + impl_fmt_trait!($name, $native, LowerHex); + impl_fmt_trait!($name, $native, UpperHex); + impl_fmt_trait!($name, $native, Binary); + }; +} + +macro_rules! impl_ops_traits { + ($name:ident, $native:ident, "floating point number") => { + impl_ops_traits!($name, $native, @all_types); + impl_ops_traits!($name, $native, @signed_integer_floating_point); + + impl PartialOrd for $name { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option { + self.get().partial_cmp(&other.get()) + } + } + }; + ($name:ident, $native:ident, "unsigned integer") => { + impl_ops_traits!($name, $native, @signed_unsigned_integer); + impl_ops_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, "signed integer") => { + impl_ops_traits!($name, $native, @signed_unsigned_integer); + impl_ops_traits!($name, $native, @signed_integer_floating_point); + impl_ops_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, @signed_unsigned_integer) => { + impl_ops_traits!(@without_byteorder_swap $name, $native, BitAnd, bitand, BitAndAssign, bitand_assign); + impl_ops_traits!(@without_byteorder_swap $name, $native, BitOr, bitor, BitOrAssign, bitor_assign); + impl_ops_traits!(@without_byteorder_swap $name, $native, BitXor, bitxor, BitXorAssign, bitxor_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Shl, shl, ShlAssign, shl_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Shr, shr, ShrAssign, shr_assign); + + impl core::ops::Not for $name { + type Output = $name; + + #[inline(always)] + fn not(self) -> $name { + let self_native = $native::from_ne_bytes(self.0); + $name((!self_native).to_ne_bytes(), PhantomData) + } + } + + impl PartialOrd for $name { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for $name { + #[inline(always)] + fn cmp(&self, other: &Self) -> Ordering { + self.get().cmp(&other.get()) + } + } + + impl PartialOrd<$native> for $name { + #[inline(always)] + fn partial_cmp(&self, other: &$native) -> Option { + self.get().partial_cmp(other) + } + } + }; + ($name:ident, $native:ident, @signed_integer_floating_point) => { + impl core::ops::Neg for $name { + type Output = $name; + + #[inline(always)] + fn neg(self) -> $name { + let self_native: $native = self.get(); + #[allow(clippy::arithmetic_side_effects)] + $name::::new(-self_native) + } + } + }; + ($name:ident, $native:ident, @all_types) => { + impl_ops_traits!(@with_byteorder_swap $name, $native, Add, add, AddAssign, add_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Div, div, DivAssign, div_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Mul, mul, MulAssign, mul_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Rem, rem, RemAssign, rem_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Sub, sub, SubAssign, sub_assign); + }; + (@with_byteorder_swap $name:ident, $native:ident, $trait:ident, $method:ident, $trait_assign:ident, $method_assign:ident) => { + impl core::ops::$trait<$name> for $name { + type Output = $name; + + #[inline(always)] + fn $method(self, rhs: $name) -> $name { + let self_native: $native = self.get(); + let rhs_native: $native = rhs.get(); + let result_native = core::ops::$trait::$method(self_native, rhs_native); + $name::::new(result_native) + } + } + + impl core::ops::$trait<$name> for $native { + type Output = $name; + + #[inline(always)] + fn $method(self, rhs: $name) -> $name { + let rhs_native: $native = rhs.get(); + let result_native = core::ops::$trait::$method(self, rhs_native); + $name::::new(result_native) + } + } + + impl core::ops::$trait<$native> for $name { + type Output = $name; + + #[inline(always)] + fn $method(self, rhs: $native) -> $name { + let self_native: $native = self.get(); + let result_native = core::ops::$trait::$method(self_native, rhs); + $name::::new(result_native) + } + } + + impl core::ops::$trait_assign<$name> for $name { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + + impl core::ops::$trait_assign<$name> for $native { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name) { + let rhs_native: $native = rhs.get(); + *self = core::ops::$trait::$method(*self, rhs_native); + } + } + + impl core::ops::$trait_assign<$native> for $name { + #[inline(always)] + fn $method_assign(&mut self, rhs: $native) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + }; + // Implement traits in terms of the same trait on the native type, but + // without performing a byte order swap when both operands are byteorder + // types. This only works for bitwise operations like `&`, `|`, etc. + // + // When only one operand is a byteorder type, we still need to perform a + // byteorder swap. + (@without_byteorder_swap $name:ident, $native:ident, $trait:ident, $method:ident, $trait_assign:ident, $method_assign:ident) => { + impl core::ops::$trait<$name> for $name { + type Output = $name; + + #[inline(always)] + fn $method(self, rhs: $name) -> $name { + let self_native = $native::from_ne_bytes(self.0); + let rhs_native = $native::from_ne_bytes(rhs.0); + let result_native = core::ops::$trait::$method(self_native, rhs_native); + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl core::ops::$trait<$name> for $native { + type Output = $name; + + #[inline(always)] + fn $method(self, rhs: $name) -> $name { + // No runtime cost - just byte packing + let rhs_native = $native::from_ne_bytes(rhs.0); + // (Maybe) runtime cost - byte order swap + let slf_byteorder = $name::::new(self); + // No runtime cost - just byte packing + let slf_native = $native::from_ne_bytes(slf_byteorder.0); + // Runtime cost - perform the operation + let result_native = core::ops::$trait::$method(slf_native, rhs_native); + // No runtime cost - just byte unpacking + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl core::ops::$trait<$native> for $name { + type Output = $name; + + #[inline(always)] + fn $method(self, rhs: $native) -> $name { + // (Maybe) runtime cost - byte order swap + let rhs_byteorder = $name::::new(rhs); + // No runtime cost - just byte packing + let rhs_native = $native::from_ne_bytes(rhs_byteorder.0); + // No runtime cost - just byte packing + let slf_native = $native::from_ne_bytes(self.0); + // Runtime cost - perform the operation + let result_native = core::ops::$trait::$method(slf_native, rhs_native); + // No runtime cost - just byte unpacking + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl core::ops::$trait_assign<$name> for $name { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + + impl core::ops::$trait_assign<$name> for $native { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name) { + // (Maybe) runtime cost - byte order swap + let rhs_native = rhs.get(); + // Runtime cost - perform the operation + *self = core::ops::$trait::$method(*self, rhs_native); + } + } + + impl core::ops::$trait_assign<$native> for $name { + #[inline(always)] + fn $method_assign(&mut self, rhs: $native) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + }; +} + +macro_rules! doc_comment { + ($x:expr, $($tt:tt)*) => { + #[doc = $x] + $($tt)* + }; +} + +macro_rules! define_max_value_constant { + ($name:ident, $bytes:expr, "unsigned integer") => { + /// The maximum value. + /// + /// This constant should be preferred to constructing a new value using + /// `new`, as `new` may perform an endianness swap depending on the + /// endianness `O` and the endianness of the platform. + pub const MAX_VALUE: $name = $name([0xFFu8; $bytes], PhantomData); + }; + // We don't provide maximum and minimum value constants for signed values + // and floats because there's no way to do it generically - it would require + // a different value depending on the value of the `ByteOrder` type + // parameter. Currently, one workaround would be to provide implementations + // for concrete implementations of that trait. In the long term, if we are + // ever able to make the `new` constructor a const fn, we could use that + // instead. + ($name:ident, $bytes:expr, "signed integer") => {}; + ($name:ident, $bytes:expr, "floating point number") => {}; +} + +macro_rules! define_type { + ( + $article:ident, + $description:expr, + $name:ident, + $native:ident, + $bits:expr, + $bytes:expr, + $from_be_fn:path, + $to_be_fn:path, + $from_le_fn:path, + $to_le_fn:path, + $number_kind:tt, + [$($larger_native:ty),*], + [$($larger_native_try:ty),*], + [$($larger_byteorder:ident),*], + [$($larger_byteorder_try:ident),*] + ) => { + doc_comment! { + concat!($description, " stored in a given byte order. + +`", stringify!($name), "` is like the native `", stringify!($native), "` type with +two major differences: First, it has no alignment requirement (its alignment is 1). +Second, the endianness of its memory layout is given by the type parameter `O`, +which can be any type which implements [`ByteOrder`]. In particular, this refers +to [`BigEndian`], [`LittleEndian`], [`NativeEndian`], and [`NetworkEndian`]. + +", stringify!($article), " `", stringify!($name), "` can be constructed using +the [`new`] method, and its contained value can be obtained as a native +`",stringify!($native), "` using the [`get`] method, or updated in place with +the [`set`] method. In all cases, if the endianness `O` is not the same as the +endianness of the current platform, an endianness swap will be performed in +order to uphold the invariants that a) the layout of `", stringify!($name), "` +has endianness `O` and that, b) the layout of `", stringify!($native), "` has +the platform's native endianness. + +`", stringify!($name), "` implements [`FromBytes`], [`IntoBytes`], and [`Unaligned`], +making it useful for parsing and serialization. See the module documentation for an +example of how it can be used for parsing UDP packets. + +[`new`]: crate::byteorder::", stringify!($name), "::new +[`get`]: crate::byteorder::", stringify!($name), "::get +[`set`]: crate::byteorder::", stringify!($name), "::set +[`FromBytes`]: crate::FromBytes +[`IntoBytes`]: crate::IntoBytes +[`Unaligned`]: crate::Unaligned"), + #[derive(Copy, Clone, Eq, PartialEq, Hash)] + #[cfg_attr(any(feature = "derive", test), derive(KnownLayout, Immutable, FromBytes, IntoBytes, Unaligned))] + #[repr(transparent)] + pub struct $name([u8; $bytes], PhantomData); + } + + #[cfg(not(any(feature = "derive", test)))] + impl_known_layout!(O => $name); + + #[allow(unused_unsafe)] // Unused when `feature = "derive"`. + // SAFETY: `$name` is `repr(transparent)`, and so it has the same + // layout as its only non-zero field, which is a `u8` array. `u8` arrays + // are `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, + // `IntoBytes`, and `Unaligned`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + impl_or_verify!(O => Immutable for $name); + impl_or_verify!(O => TryFromBytes for $name); + impl_or_verify!(O => FromZeros for $name); + impl_or_verify!(O => FromBytes for $name); + impl_or_verify!(O => IntoBytes for $name); + impl_or_verify!(O => Unaligned for $name); + }; + + impl Default for $name { + #[inline(always)] + fn default() -> $name { + $name::ZERO + } + } + + impl $name { + /// The value zero. + /// + /// This constant should be preferred to constructing a new value + /// using `new`, as `new` may perform an endianness swap depending + /// on the endianness and platform. + pub const ZERO: $name = $name([0u8; $bytes], PhantomData); + + define_max_value_constant!($name, $bytes, $number_kind); + + /// Constructs a new value from bytes which are already in `O` byte + /// order. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn from_bytes(bytes: [u8; $bytes]) -> $name { + $name(bytes, PhantomData) + } + + /// Extracts the bytes of `self` without swapping the byte order. + /// + /// The returned bytes will be in `O` byte order. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn to_bytes(self) -> [u8; $bytes] { + self.0 + } + } + + impl $name { + maybe_const_trait_bounded_fn! { + /// Constructs a new value, possibly performing an endianness + /// swap to guarantee that the returned value has endianness + /// `O`. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn new(n: $native) -> $name { + let bytes = match O::ORDER { + Order::BigEndian => $to_be_fn(n), + Order::LittleEndian => $to_le_fn(n), + }; + + $name(bytes, PhantomData) + } + } + + maybe_const_trait_bounded_fn! { + /// Returns the value as a primitive type, possibly performing + /// an endianness swap to guarantee that the return value has + /// the endianness of the native platform. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn get(self) -> $native { + match O::ORDER { + Order::BigEndian => $from_be_fn(self.0), + Order::LittleEndian => $from_le_fn(self.0), + } + } + } + + /// Updates the value in place as a primitive type, possibly + /// performing an endianness swap to guarantee that the stored value + /// has the endianness `O`. + #[inline(always)] + pub fn set(&mut self, n: $native) { + *self = Self::new(n); + } + } + + // The reasoning behind which traits to implement here is to only + // implement traits which won't cause inference issues. Notably, + // comparison traits like PartialEq and PartialOrd tend to cause + // inference issues. + + impl From<$name> for [u8; $bytes] { + #[inline(always)] + fn from(x: $name) -> [u8; $bytes] { + x.0 + } + } + + impl From<[u8; $bytes]> for $name { + #[inline(always)] + fn from(bytes: [u8; $bytes]) -> $name { + $name(bytes, PhantomData) + } + } + + impl From<$name> for $native { + #[inline(always)] + fn from(x: $name) -> $native { + x.get() + } + } + + impl From<$native> for $name { + #[inline(always)] + fn from(x: $native) -> $name { + $name::new(x) + } + } + + $( + impl From<$name> for $larger_native { + #[inline(always)] + fn from(x: $name) -> $larger_native { + x.get().into() + } + } + )* + + $( + impl TryFrom<$larger_native_try> for $name { + type Error = TryFromIntError; + #[inline(always)] + fn try_from(x: $larger_native_try) -> Result<$name, TryFromIntError> { + $native::try_from(x).map($name::new) + } + } + )* + + $( + impl From<$name> for $larger_byteorder

{ + #[inline(always)] + fn from(x: $name) -> $larger_byteorder

{ + $larger_byteorder::new(x.get().into()) + } + } + )* + + $( + impl TryFrom<$larger_byteorder_try

> for $name { + type Error = TryFromIntError; + #[inline(always)] + fn try_from(x: $larger_byteorder_try

) -> Result<$name, TryFromIntError> { + x.get().try_into().map($name::new) + } + } + )* + + impl AsRef<[u8; $bytes]> for $name { + #[inline(always)] + fn as_ref(&self) -> &[u8; $bytes] { + &self.0 + } + } + + impl AsMut<[u8; $bytes]> for $name { + #[inline(always)] + fn as_mut(&mut self) -> &mut [u8; $bytes] { + &mut self.0 + } + } + + impl PartialEq<$name> for [u8; $bytes] { + #[inline(always)] + fn eq(&self, other: &$name) -> bool { + self.eq(&other.0) + } + } + + impl PartialEq<[u8; $bytes]> for $name { + #[inline(always)] + fn eq(&self, other: &[u8; $bytes]) -> bool { + self.0.eq(other) + } + } + + impl PartialEq<$native> for $name { + #[inline(always)] + fn eq(&self, other: &$native) -> bool { + self.get().eq(other) + } + } + + impl_fmt_traits!($name, $native, $number_kind); + impl_ops_traits!($name, $native, $number_kind); + + impl Debug for $name { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + // This results in a format like "U16(42)". + f.debug_tuple(stringify!($name)).field(&self.get()).finish() + } + } + }; +} + +define_type!( + A, + "A 16-bit unsigned integer", + U16, + u16, + 16, + 2, + u16::from_be_bytes, + u16::to_be_bytes, + u16::from_le_bytes, + u16::to_le_bytes, + "unsigned integer", + [u32, u64, u128, usize], + [u32, u64, u128, usize], + [U32, U64, U128, Usize], + [U32, U64, U128, Usize] +); +define_type!( + A, + "A 32-bit unsigned integer", + U32, + u32, + 32, + 4, + u32::from_be_bytes, + u32::to_be_bytes, + u32::from_le_bytes, + u32::to_le_bytes, + "unsigned integer", + [u64, u128], + [u64, u128], + [U64, U128], + [U64, U128] +); +define_type!( + A, + "A 64-bit unsigned integer", + U64, + u64, + 64, + 8, + u64::from_be_bytes, + u64::to_be_bytes, + u64::from_le_bytes, + u64::to_le_bytes, + "unsigned integer", + [u128], + [u128], + [U128], + [U128] +); +define_type!( + A, + "A 128-bit unsigned integer", + U128, + u128, + 128, + 16, + u128::from_be_bytes, + u128::to_be_bytes, + u128::from_le_bytes, + u128::to_le_bytes, + "unsigned integer", + [], + [], + [], + [] +); +define_type!( + A, + "A word-sized unsigned integer", + Usize, + usize, + mem::size_of::() * 8, + mem::size_of::(), + usize::from_be_bytes, + usize::to_be_bytes, + usize::from_le_bytes, + usize::to_le_bytes, + "unsigned integer", + [], + [], + [], + [] +); +define_type!( + An, + "A 16-bit signed integer", + I16, + i16, + 16, + 2, + i16::from_be_bytes, + i16::to_be_bytes, + i16::from_le_bytes, + i16::to_le_bytes, + "signed integer", + [i32, i64, i128, isize], + [i32, i64, i128, isize], + [I32, I64, I128, Isize], + [I32, I64, I128, Isize] +); +define_type!( + An, + "A 32-bit signed integer", + I32, + i32, + 32, + 4, + i32::from_be_bytes, + i32::to_be_bytes, + i32::from_le_bytes, + i32::to_le_bytes, + "signed integer", + [i64, i128], + [i64, i128], + [I64, I128], + [I64, I128] +); +define_type!( + An, + "A 64-bit signed integer", + I64, + i64, + 64, + 8, + i64::from_be_bytes, + i64::to_be_bytes, + i64::from_le_bytes, + i64::to_le_bytes, + "signed integer", + [i128], + [i128], + [I128], + [I128] +); +define_type!( + An, + "A 128-bit signed integer", + I128, + i128, + 128, + 16, + i128::from_be_bytes, + i128::to_be_bytes, + i128::from_le_bytes, + i128::to_le_bytes, + "signed integer", + [], + [], + [], + [] +); +define_type!( + An, + "A word-sized signed integer", + Isize, + isize, + mem::size_of::() * 8, + mem::size_of::(), + isize::from_be_bytes, + isize::to_be_bytes, + isize::from_le_bytes, + isize::to_le_bytes, + "signed integer", + [], + [], + [], + [] +); + +// FIXME(https://github.com/rust-lang/rust/issues/72447): Use the endianness +// conversion methods directly once those are const-stable. +macro_rules! define_float_conversion { + ($ty:ty, $bits:ident, $bytes:expr, $mod:ident) => { + mod $mod { + use super::*; + + define_float_conversion!($ty, $bits, $bytes, from_be_bytes, to_be_bytes); + define_float_conversion!($ty, $bits, $bytes, from_le_bytes, to_le_bytes); + } + }; + ($ty:ty, $bits:ident, $bytes:expr, $from:ident, $to:ident) => { + // Clippy: The suggestion of using `from_bits()` instead doesn't work + // because `from_bits` is not const-stable on our MSRV. + #[allow(clippy::unnecessary_transmutes)] + pub(crate) const fn $from(bytes: [u8; $bytes]) -> $ty { + transmute!($bits::$from(bytes)) + } + + pub(crate) const fn $to(f: $ty) -> [u8; $bytes] { + // Clippy: The suggestion of using `f.to_bits()` instead doesn't + // work because `to_bits` is not const-stable on our MSRV. + #[allow(clippy::unnecessary_transmutes)] + let bits: $bits = transmute!(f); + bits.$to() + } + }; +} + +define_float_conversion!(f32, u32, 4, f32_ext); +define_float_conversion!(f64, u64, 8, f64_ext); + +define_type!( + An, + "A 32-bit floating point number", + F32, + f32, + 32, + 4, + f32_ext::from_be_bytes, + f32_ext::to_be_bytes, + f32_ext::from_le_bytes, + f32_ext::to_le_bytes, + "floating point number", + [f64], + [], + [F64], + [] +); +define_type!( + An, + "A 64-bit floating point number", + F64, + f64, + 64, + 8, + f64_ext::from_be_bytes, + f64_ext::to_be_bytes, + f64_ext::from_le_bytes, + f64_ext::to_le_bytes, + "floating point number", + [], + [], + [], + [] +); + +macro_rules! module { + ($name:ident, $trait:ident, $endianness_str:expr) => { + /// Numeric primitives stored in + #[doc = $endianness_str] + /// byte order. + pub mod $name { + use super::$trait; + + module!(@ty U16, $trait, "16-bit unsigned integer", $endianness_str); + module!(@ty U32, $trait, "32-bit unsigned integer", $endianness_str); + module!(@ty U64, $trait, "64-bit unsigned integer", $endianness_str); + module!(@ty U128, $trait, "128-bit unsigned integer", $endianness_str); + module!(@ty I16, $trait, "16-bit signed integer", $endianness_str); + module!(@ty I32, $trait, "32-bit signed integer", $endianness_str); + module!(@ty I64, $trait, "64-bit signed integer", $endianness_str); + module!(@ty I128, $trait, "128-bit signed integer", $endianness_str); + module!(@ty F32, $trait, "32-bit floating point number", $endianness_str); + module!(@ty F64, $trait, "64-bit floating point number", $endianness_str); + } + }; + (@ty $ty:ident, $trait:ident, $desc_str:expr, $endianness_str:expr) => { + /// A + #[doc = $desc_str] + /// stored in + #[doc = $endianness_str] + /// byte order. + pub type $ty = crate::byteorder::$ty<$trait>; + }; +} + +module!(big_endian, BigEndian, "big-endian"); +module!(little_endian, LittleEndian, "little-endian"); +module!(network_endian, NetworkEndian, "network-endian"); +module!(native_endian, NativeEndian, "native-endian"); + +#[cfg(any(test, kani))] +mod tests { + use super::*; + + #[cfg(not(kani))] + mod compatibility { + pub(super) use rand::{ + distributions::{Distribution, Standard}, + rngs::SmallRng, + Rng, SeedableRng, + }; + + pub(crate) trait Arbitrary {} + + impl Arbitrary for T {} + } + + #[cfg(kani)] + mod compatibility { + pub(crate) use kani::Arbitrary; + + pub(crate) struct SmallRng; + + impl SmallRng { + pub(crate) fn seed_from_u64(_state: u64) -> Self { + Self + } + } + + pub(crate) trait Rng { + fn sample>(&mut self, _distr: D) -> T + where + T: Arbitrary, + { + kani::any() + } + } + + impl Rng for SmallRng {} + + pub(crate) trait Distribution {} + impl Distribution for U {} + + pub(crate) struct Standard; + } + + use compatibility::*; + + // A native integer type (u16, i32, etc). + trait Native: Arbitrary + FromBytes + IntoBytes + Immutable + Copy + PartialEq + Debug { + const ZERO: Self; + const MAX_VALUE: Self; + + type Distribution: Distribution; + const DIST: Self::Distribution; + + fn rand(rng: &mut R) -> Self { + rng.sample(Self::DIST) + } + + #[cfg_attr(kani, allow(unused))] + fn checked_add(self, rhs: Self) -> Option; + + #[cfg_attr(kani, allow(unused))] + fn checked_div(self, rhs: Self) -> Option; + + #[cfg_attr(kani, allow(unused))] + fn checked_mul(self, rhs: Self) -> Option; + + #[cfg_attr(kani, allow(unused))] + fn checked_rem(self, rhs: Self) -> Option; + + #[cfg_attr(kani, allow(unused))] + fn checked_sub(self, rhs: Self) -> Option; + + #[cfg_attr(kani, allow(unused))] + fn checked_shl(self, rhs: Self) -> Option; + + #[cfg_attr(kani, allow(unused))] + fn checked_shr(self, rhs: Self) -> Option; + + fn is_nan(self) -> bool; + + /// For `f32` and `f64`, NaN values are not considered equal to + /// themselves. This method is like `assert_eq!`, but it treats NaN + /// values as equal. + fn assert_eq_or_nan(self, other: Self) { + let slf = (!self.is_nan()).then(|| self); + let other = (!other.is_nan()).then(|| other); + assert_eq!(slf, other); + } + } + + trait ByteArray: + FromBytes + IntoBytes + Immutable + Copy + AsRef<[u8]> + AsMut<[u8]> + Debug + Default + Eq + { + /// Invert the order of the bytes in the array. + fn invert(self) -> Self; + } + + trait ByteOrderType: + FromBytes + IntoBytes + Unaligned + Copy + Eq + Debug + Hash + From + { + type Native: Native; + type ByteArray: ByteArray; + + const ZERO: Self; + + fn new(native: Self::Native) -> Self; + fn get(self) -> Self::Native; + fn set(&mut self, native: Self::Native); + fn from_bytes(bytes: Self::ByteArray) -> Self; + fn into_bytes(self) -> Self::ByteArray; + + /// For `f32` and `f64`, NaN values are not considered equal to + /// themselves. This method is like `assert_eq!`, but it treats NaN + /// values as equal. + fn assert_eq_or_nan(self, other: Self) { + let slf = (!self.get().is_nan()).then(|| self); + let other = (!other.get().is_nan()).then(|| other); + assert_eq!(slf, other); + } + } + + trait ByteOrderTypeUnsigned: ByteOrderType { + const MAX_VALUE: Self; + } + + macro_rules! impl_byte_array { + ($bytes:expr) => { + impl ByteArray for [u8; $bytes] { + fn invert(mut self) -> [u8; $bytes] { + self.reverse(); + self + } + } + }; + } + + impl_byte_array!(2); + impl_byte_array!(4); + impl_byte_array!(8); + impl_byte_array!(16); + + macro_rules! impl_byte_order_type_unsigned { + ($name:ident, unsigned) => { + impl ByteOrderTypeUnsigned for $name { + const MAX_VALUE: $name = $name::MAX_VALUE; + } + }; + ($name:ident, signed) => {}; + } + + macro_rules! impl_traits { + ($name:ident, $native:ident, $sign:ident $(, @$float:ident)?) => { + impl Native for $native { + // For some types, `0 as $native` is required (for example, when + // `$native` is a floating-point type; `0` is an integer), but + // for other types, it's a trivial cast. In all cases, Clippy + // thinks it's dangerous. + #[allow(trivial_numeric_casts, clippy::as_conversions)] + const ZERO: $native = 0 as $native; + const MAX_VALUE: $native = $native::MAX; + + type Distribution = Standard; + const DIST: Standard = Standard; + + impl_traits!(@float_dependent_methods $(@$float)?); + } + + impl ByteOrderType for $name { + type Native = $native; + type ByteArray = [u8; mem::size_of::<$native>()]; + + const ZERO: $name = $name::ZERO; + + fn new(native: $native) -> $name { + $name::new(native) + } + + fn get(self) -> $native { + $name::get(self) + } + + fn set(&mut self, native: $native) { + $name::set(self, native) + } + + fn from_bytes(bytes: [u8; mem::size_of::<$native>()]) -> $name { + $name::from(bytes) + } + + fn into_bytes(self) -> [u8; mem::size_of::<$native>()] { + <[u8; mem::size_of::<$native>()]>::from(self) + } + } + + impl_byte_order_type_unsigned!($name, $sign); + }; + (@float_dependent_methods) => { + fn checked_add(self, rhs: Self) -> Option { self.checked_add(rhs) } + fn checked_div(self, rhs: Self) -> Option { self.checked_div(rhs) } + fn checked_mul(self, rhs: Self) -> Option { self.checked_mul(rhs) } + fn checked_rem(self, rhs: Self) -> Option { self.checked_rem(rhs) } + fn checked_sub(self, rhs: Self) -> Option { self.checked_sub(rhs) } + fn checked_shl(self, rhs: Self) -> Option { self.checked_shl(rhs.try_into().unwrap_or(u32::MAX)) } + fn checked_shr(self, rhs: Self) -> Option { self.checked_shr(rhs.try_into().unwrap_or(u32::MAX)) } + fn is_nan(self) -> bool { false } + }; + (@float_dependent_methods @float) => { + fn checked_add(self, rhs: Self) -> Option { Some(self + rhs) } + fn checked_div(self, rhs: Self) -> Option { Some(self / rhs) } + fn checked_mul(self, rhs: Self) -> Option { Some(self * rhs) } + fn checked_rem(self, rhs: Self) -> Option { Some(self % rhs) } + fn checked_sub(self, rhs: Self) -> Option { Some(self - rhs) } + fn checked_shl(self, _rhs: Self) -> Option { unimplemented!() } + fn checked_shr(self, _rhs: Self) -> Option { unimplemented!() } + fn is_nan(self) -> bool { self.is_nan() } + }; + } + + impl_traits!(U16, u16, unsigned); + impl_traits!(U32, u32, unsigned); + impl_traits!(U64, u64, unsigned); + impl_traits!(U128, u128, unsigned); + impl_traits!(Usize, usize, unsigned); + impl_traits!(I16, i16, signed); + impl_traits!(I32, i32, signed); + impl_traits!(I64, i64, signed); + impl_traits!(I128, i128, signed); + impl_traits!(Isize, isize, unsigned); + impl_traits!(F32, f32, signed, @float); + impl_traits!(F64, f64, signed, @float); + + macro_rules! call_for_unsigned_types { + ($fn:ident, $byteorder:ident) => { + $fn::>(); + $fn::>(); + $fn::>(); + $fn::>(); + $fn::>(); + }; + } + + macro_rules! call_for_signed_types { + ($fn:ident, $byteorder:ident) => { + $fn::>(); + $fn::>(); + $fn::>(); + $fn::>(); + $fn::>(); + }; + } + + macro_rules! call_for_float_types { + ($fn:ident, $byteorder:ident) => { + $fn::>(); + $fn::>(); + }; + } + + macro_rules! call_for_all_types { + ($fn:ident, $byteorder:ident) => { + call_for_unsigned_types!($fn, $byteorder); + call_for_signed_types!($fn, $byteorder); + call_for_float_types!($fn, $byteorder); + }; + } + + #[cfg(target_endian = "big")] + type NonNativeEndian = LittleEndian; + #[cfg(target_endian = "little")] + type NonNativeEndian = BigEndian; + + // We use a `u64` seed so that we can use `SeedableRng::seed_from_u64`. + // `SmallRng`'s `SeedableRng::Seed` differs by platform, so if we wanted to + // call `SeedableRng::from_seed`, which takes a `Seed`, we would need + // conditional compilation by `target_pointer_width`. + const RNG_SEED: u64 = 0x7A03CAE2F32B5B8F; + + const RAND_ITERS: usize = if cfg!(any(miri, kani)) { + // The tests below which use this constant used to take a very long time + // on Miri, which slows down local development and CI jobs. We're not + // using Miri to check for the correctness of our code, but rather its + // soundness, and at least in the context of these particular tests, a + // single loop iteration is just as good for surfacing UB as multiple + // iterations are. + // + // As of the writing of this comment, here's one set of measurements: + // + // $ # RAND_ITERS == 1 + // $ cargo miri test -- -Z unstable-options --report-time endian + // test byteorder::tests::test_native_endian ... ok <0.049s> + // test byteorder::tests::test_non_native_endian ... ok <0.061s> + // + // $ # RAND_ITERS == 1024 + // $ cargo miri test -- -Z unstable-options --report-time endian + // test byteorder::tests::test_native_endian ... ok <25.716s> + // test byteorder::tests::test_non_native_endian ... ok <38.127s> + 1 + } else { + 1024 + }; + + #[test] + fn test_const_methods() { + use big_endian::*; + + #[rustversion::since(1.61.0)] + const _U: U16 = U16::new(0); + #[rustversion::since(1.61.0)] + const _NATIVE: u16 = _U.get(); + const _FROM_BYTES: U16 = U16::from_bytes([0, 1]); + const _BYTES: [u8; 2] = _FROM_BYTES.to_bytes(); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_zero() { + fn test_zero() { + assert_eq!(T::ZERO.get(), T::Native::ZERO); + } + + call_for_all_types!(test_zero, NativeEndian); + call_for_all_types!(test_zero, NonNativeEndian); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_max_value() { + fn test_max_value() { + assert_eq!(T::MAX_VALUE.get(), T::Native::MAX_VALUE); + } + + call_for_unsigned_types!(test_max_value, NativeEndian); + call_for_unsigned_types!(test_max_value, NonNativeEndian); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_endian() { + fn test(invert: bool) { + let mut r = SmallRng::seed_from_u64(RNG_SEED); + for _ in 0..RAND_ITERS { + let native = T::Native::rand(&mut r); + let mut bytes = T::ByteArray::default(); + bytes.as_mut_bytes().copy_from_slice(native.as_bytes()); + if invert { + bytes = bytes.invert(); + } + let mut from_native = T::new(native); + let from_bytes = T::from_bytes(bytes); + + from_native.assert_eq_or_nan(from_bytes); + from_native.get().assert_eq_or_nan(native); + from_bytes.get().assert_eq_or_nan(native); + + assert_eq!(from_native.into_bytes(), bytes); + assert_eq!(from_bytes.into_bytes(), bytes); + + let updated = T::Native::rand(&mut r); + from_native.set(updated); + from_native.get().assert_eq_or_nan(updated); + } + } + + fn test_native() { + test::(false); + } + + fn test_non_native() { + test::(true); + } + + call_for_all_types!(test_native, NativeEndian); + call_for_all_types!(test_non_native, NonNativeEndian); + } + + #[test] + fn test_ops_impls() { + // Test implementations of traits in `core::ops`. Some of these are + // fairly banal, but some are optimized to perform the operation without + // swapping byte order (namely, bit-wise operations which are identical + // regardless of byte order). These are important to test, and while + // we're testing those anyway, it's trivial to test all of the impls. + + fn test( + op_t_t: FTT, + op_t_n: FTN, + op_n_t: FNT, + op_n_n: FNN, + op_n_n_checked: Option, + op_assign: Option<(FATT, FATN, FANT)>, + ) where + T: ByteOrderType, + FTT: Fn(T, T) -> T, + FTN: Fn(T, T::Native) -> T, + FNT: Fn(T::Native, T) -> T, + FNN: Fn(T::Native, T::Native) -> T::Native, + FNNChecked: Fn(T::Native, T::Native) -> Option, + FATT: Fn(&mut T, T), + FATN: Fn(&mut T, T::Native), + FANT: Fn(&mut T::Native, T), + { + let mut r = SmallRng::seed_from_u64(RNG_SEED); + for _ in 0..RAND_ITERS { + let n0 = T::Native::rand(&mut r); + let n1 = T::Native::rand(&mut r); + let t0 = T::new(n0); + let t1 = T::new(n1); + + // If this operation would overflow/underflow, skip it rather + // than attempt to catch and recover from panics. + if matches!(&op_n_n_checked, Some(checked) if checked(n0, n1).is_none()) { + continue; + } + + let t_t_res = op_t_t(t0, t1); + let t_n_res = op_t_n(t0, n1); + let n_t_res = op_n_t(n0, t1); + let n_n_res = op_n_n(n0, n1); + + // For `f32` and `f64`, NaN values are not considered equal to + // themselves. We store `Option`/`Option` and store + // NaN as `None` so they can still be compared. + let val_or_none = |t: T| (!T::Native::is_nan(t.get())).then(|| t.get()); + let t_t_res = val_or_none(t_t_res); + let t_n_res = val_or_none(t_n_res); + let n_t_res = val_or_none(n_t_res); + let n_n_res = (!T::Native::is_nan(n_n_res)).then(|| n_n_res); + assert_eq!(t_t_res, n_n_res); + assert_eq!(t_n_res, n_n_res); + assert_eq!(n_t_res, n_n_res); + + if let Some((op_assign_t_t, op_assign_t_n, op_assign_n_t)) = &op_assign { + let mut t_t_res = t0; + op_assign_t_t(&mut t_t_res, t1); + let mut t_n_res = t0; + op_assign_t_n(&mut t_n_res, n1); + let mut n_t_res = n0; + op_assign_n_t(&mut n_t_res, t1); + + // For `f32` and `f64`, NaN values are not considered equal to + // themselves. We store `Option`/`Option` and store + // NaN as `None` so they can still be compared. + let t_t_res = val_or_none(t_t_res); + let t_n_res = val_or_none(t_n_res); + let n_t_res = (!T::Native::is_nan(n_t_res)).then(|| n_t_res); + assert_eq!(t_t_res, n_n_res); + assert_eq!(t_n_res, n_n_res); + assert_eq!(n_t_res, n_n_res); + } + } + } + + macro_rules! test { + ( + @binary + $trait:ident, + $method:ident $([$checked_method:ident])?, + $trait_assign:ident, + $method_assign:ident, + $($call_for_macros:ident),* + ) => {{ + fn t() + where + T: ByteOrderType, + T: core::ops::$trait, + T: core::ops::$trait, + T::Native: core::ops::$trait, + T::Native: core::ops::$trait, + + T: core::ops::$trait_assign, + T: core::ops::$trait_assign, + T::Native: core::ops::$trait_assign, + T::Native: core::ops::$trait_assign, + { + test::( + core::ops::$trait::$method, + core::ops::$trait::$method, + core::ops::$trait::$method, + core::ops::$trait::$method, + { + #[allow(unused_mut, unused_assignments)] + let mut op_native_checked = None:: Option>; + $( + op_native_checked = Some(T::Native::$checked_method); + )? + op_native_checked + }, + Some(( + >::$method_assign, + >::$method_assign, + >::$method_assign + )), + ); + } + + $( + $call_for_macros!(t, NativeEndian); + $call_for_macros!(t, NonNativeEndian); + )* + }}; + ( + @unary + $trait:ident, + $method:ident, + $($call_for_macros:ident),* + ) => {{ + fn t() + where + T: ByteOrderType, + T: core::ops::$trait, + T::Native: core::ops::$trait, + { + test::( + |slf, _rhs| core::ops::$trait::$method(slf), + |slf, _rhs| core::ops::$trait::$method(slf), + |slf, _rhs| core::ops::$trait::$method(slf).into(), + |slf, _rhs| core::ops::$trait::$method(slf), + None:: Option>, + None::<(fn(&mut T, T), fn(&mut T, T::Native), fn(&mut T::Native, T))>, + ); + } + + $( + $call_for_macros!(t, NativeEndian); + $call_for_macros!(t, NonNativeEndian); + )* + }}; + } + + test!(@binary Add, add[checked_add], AddAssign, add_assign, call_for_all_types); + test!(@binary Div, div[checked_div], DivAssign, div_assign, call_for_all_types); + test!(@binary Mul, mul[checked_mul], MulAssign, mul_assign, call_for_all_types); + test!(@binary Rem, rem[checked_rem], RemAssign, rem_assign, call_for_all_types); + test!(@binary Sub, sub[checked_sub], SubAssign, sub_assign, call_for_all_types); + + test!(@binary BitAnd, bitand, BitAndAssign, bitand_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary BitOr, bitor, BitOrAssign, bitor_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary BitXor, bitxor, BitXorAssign, bitxor_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary Shl, shl[checked_shl], ShlAssign, shl_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary Shr, shr[checked_shr], ShrAssign, shr_assign, call_for_unsigned_types, call_for_signed_types); + + test!(@unary Not, not, call_for_signed_types, call_for_unsigned_types); + test!(@unary Neg, neg, call_for_signed_types, call_for_float_types); + } + + #[test] + fn test_debug_impl() { + // Ensure that Debug applies format options to the inner value. + let val = U16::::new(10); + assert_eq!(format!("{:?}", val), "U16(10)"); + assert_eq!(format!("{:03?}", val), "U16(010)"); + assert_eq!(format!("{:x?}", val), "U16(a)"); + } + + #[test] + fn test_byteorder_traits_coverage() { + let val_be = U16::::from_bytes([0, 1]); + let val_le = U16::::from_bytes([1, 0]); + + assert_eq!(val_be.get(), 1); + assert_eq!(val_le.get(), 1); + + // Debug + assert_eq!(format!("{:?}", val_be), "U16(1)"); + assert_eq!(format!("{:?}", val_le), "U16(1)"); + + // PartialOrd, Ord with same type + assert!(val_be >= val_be); + assert!(val_be <= val_be); + assert_eq!(val_be.cmp(&val_be), core::cmp::Ordering::Equal); + + // PartialOrd with native + assert!(val_be == 1u16); + assert!(val_be >= 1u16); + + // Default + let default_be: U16 = Default::default(); + assert_eq!(default_be.get(), 0); + + // I16 + let val_be_i16 = I16::::from_bytes([0, 1]); + assert_eq!(val_be_i16.get(), 1); + assert_eq!(format!("{:?}", val_be_i16), "I16(1)"); + assert_eq!(val_be_i16.cmp(&val_be_i16), core::cmp::Ordering::Equal); + } +} diff --git a/rust/zerocopy/src/deprecated.rs b/rust/zerocopy/src/deprecated.rs new file mode 100644 index 000000000000..61b29aa5f485 --- /dev/null +++ b/rust/zerocopy/src/deprecated.rs @@ -0,0 +1,279 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Deprecated items. These are kept separate so that they don't clutter up +//! other modules. + +use super::*; + +impl Ref +where + B: ByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new(bytes: B) -> Option> { + Self::from_bytes(bytes).ok() + } +} + +impl Ref +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_prefix`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_from_prefix(bytes: B) -> Option<(Ref, B)> { + Self::from_prefix(bytes).ok() + } +} + +impl Ref +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_suffix`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_from_suffix(bytes: B) -> Option<(B, Ref)> { + Self::from_suffix(bytes).ok() + } +} + +impl Ref +where + B: ByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_bytes`; for `T: Unaligned`, the returned `CastError` implements `Into`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned(bytes: B) -> Option> { + Self::from_bytes(bytes).ok() + } +} + +impl Ref +where + B: SplitByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_prefix`; for `T: Unaligned`, the returned `CastError` implements `Into`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned_from_prefix(bytes: B) -> Option<(Ref, B)> { + Self::from_prefix(bytes).ok() + } +} + +impl Ref +where + B: SplitByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_suffix`; for `T: Unaligned`, the returned `CastError` implements `Into`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned_from_suffix(bytes: B) -> Option<(B, Ref)> { + Self::from_suffix(bytes).ok() + } +} + +impl Ref +where + B: ByteSlice, + T: Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::from_bytes` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice(bytes: B) -> Option> { + Self::from_bytes(bytes).ok() + } +} + +impl Ref +where + B: ByteSlice, + T: Unaligned + Immutable, +{ + #[deprecated( + since = "0.8.0", + note = "`Ref::from_bytes` now supports slices; for `T: Unaligned`, the returned `CastError` implements `Into`" + )] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_unaligned(bytes: B) -> Option> { + Ref::from_bytes(bytes).ok() + } +} + +impl<'a, B, T> Ref +where + B: 'a + IntoByteSlice<'a>, + T: FromBytes + Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::into_ref` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn into_slice(self) -> &'a [T] { + Ref::into_ref(self) + } +} + +impl<'a, B, T> Ref +where + B: 'a + IntoByteSliceMut<'a>, + T: FromBytes + IntoBytes + Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::into_mut` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn into_mut_slice(self) -> &'a mut [T] { + Ref::into_mut(self) + } +} + +impl Ref +where + B: SplitByteSlice, + T: Immutable, +{ + #[deprecated(since = "0.8.0", note = "replaced by `Ref::from_prefix_with_elems`")] + #[must_use = "has no side effects"] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_from_prefix(bytes: B, count: usize) -> Option<(Ref, B)> { + Ref::from_prefix_with_elems(bytes, count).ok() + } + + #[deprecated(since = "0.8.0", note = "replaced by `Ref::from_suffix_with_elems`")] + #[must_use = "has no side effects"] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_from_suffix(bytes: B, count: usize) -> Option<(B, Ref)> { + Ref::from_suffix_with_elems(bytes, count).ok() + } +} + +impl Ref +where + B: SplitByteSlice, + T: Unaligned + Immutable, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_prefix_with_elems`; for `T: Unaligned`, the returned `CastError` implements `Into`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_slice_unaligned_from_prefix(bytes: B, count: usize) -> Option<(Ref, B)> { + Ref::from_prefix_with_elems(bytes, count).ok() + } + + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_suffix_with_elems`; for `T: Unaligned`, the returned `CastError` implements `Into`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_slice_unaligned_from_suffix(bytes: B, count: usize) -> Option<(B, Ref)> { + Ref::from_suffix_with_elems(bytes, count).ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[allow(deprecated)] + fn test_deprecated_ref_methods() { + let bytes = &[0u8; 1][..]; + let bytes_slice = &[0u8; 4][..]; + + let r: Option> = Ref::new(bytes); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], u8>, &[u8])> = Ref::new_from_prefix(bytes); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], u8>)> = Ref::new_from_suffix(bytes); + assert!(r.is_some()); + + let r: Option> = Ref::new_unaligned(bytes); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], u8>, &[u8])> = Ref::new_unaligned_from_prefix(bytes); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], u8>)> = Ref::new_unaligned_from_suffix(bytes); + assert!(r.is_some()); + + let r: Option> = Ref::new_slice(bytes_slice); + assert!(r.is_some()); + + let r: Option> = Ref::new_slice_unaligned(bytes_slice); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], [u8]>, &[u8])> = Ref::new_slice_from_prefix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], [u8]>)> = Ref::new_slice_from_suffix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], [u8]>, &[u8])> = + Ref::new_slice_unaligned_from_prefix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], [u8]>)> = + Ref::new_slice_unaligned_from_suffix(bytes_slice, 1); + assert!(r.is_some()); + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_into_slice() { + let bytes = &[0u8; 4][..]; + let r: Ref<&[u8], [u8]> = Ref::from_bytes(bytes).unwrap(); + let slice: &[u8] = r.into_slice(); + assert_eq!(slice.len(), 4); + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_into_mut_slice() { + let mut bytes = [0u8; 4]; + let r: Ref<&mut [u8], [u8]> = Ref::from_bytes(&mut bytes[..]).unwrap(); + let slice: &mut [u8] = r.into_mut_slice(); + assert_eq!(slice.len(), 4); + } +} diff --git a/rust/zerocopy/src/error.rs b/rust/zerocopy/src/error.rs new file mode 100644 index 000000000000..05fdfe3b91f8 --- /dev/null +++ b/rust/zerocopy/src/error.rs @@ -0,0 +1,1348 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Types related to error reporting. +//! +//! ## Single failure mode errors +//! +//! Generally speaking, zerocopy's conversions may fail for one of up to three +//! reasons: +//! - [`AlignmentError`]: the conversion source was improperly aligned +//! - [`SizeError`]: the conversion source was of incorrect size +//! - [`ValidityError`]: the conversion source contained invalid data +//! +//! Methods that only have one failure mode, like +//! [`FromBytes::read_from_bytes`], return that mode's corresponding error type +//! directly. +//! +//! ## Compound errors +//! +//! Conversion methods that have either two or three possible failure modes +//! return one of these error types: +//! - [`CastError`]: the error type of reference conversions +//! - [`TryCastError`]: the error type of fallible reference conversions +//! - [`TryReadError`]: the error type of fallible read conversions +//! +//! ## [`Unaligned`] destination types +//! +//! For [`Unaligned`] destination types, alignment errors are impossible. All +//! compound error types support infallibly discarding the alignment error via +//! [`From`] so long as `Dst: Unaligned`. For example, see [`>::from`][size-error-from]. +//! +//! [size-error-from]: struct.SizeError.html#method.from-1 +//! +//! ## Accessing the conversion source +//! +//! All error types provide an `into_src` method that converts the error into +//! the source value underlying the failed conversion. +//! +//! ## Display formatting +//! +//! All error types provide a `Display` implementation that produces a +//! human-readable error message. When `debug_assertions` are enabled, these +//! error messages are verbose and may include potentially sensitive +//! information, including: +//! +//! - the names of the involved types +//! - the sizes of the involved types +//! - the addresses of the involved types +//! - the contents of the involved types +//! +//! When `debug_assertions` are disabled (as is default for `release` builds), +//! such potentially sensitive information is excluded. +//! +//! In the future, we may support manually configuring this behavior. If you are +//! interested in this feature, [let us know on GitHub][issue-1457] so we know +//! to prioritize it. +//! +//! [issue-1457]: https://github.com/google/zerocopy/issues/1457 +//! +//! ## Validation order +//! +//! Our conversion methods typically check alignment, then size, then bit +//! validity. However, we do not guarantee that this is always the case, and +//! this behavior may change between releases. +//! +//! ## `Send`, `Sync`, and `'static` +//! +//! Our error types are `Send`, `Sync`, and `'static` when their `Src` parameter +//! is `Send`, `Sync`, or `'static`, respectively. This can cause issues when an +//! error is sent or synchronized across threads; e.g.: +//! +//! ```compile_fail,E0515 +//! use zerocopy::*; +//! +//! let result: SizeError<&[u8], u32> = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source).unwrap_err() +//! }).join().unwrap(); +//! ``` +//! +//! To work around this, use [`map_src`][CastError::map_src] to convert the +//! source parameter to an unproblematic type; e.g.: +//! +//! ``` +//! use zerocopy::*; +//! +//! let result: SizeError<(), u32> = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source).unwrap_err() +//! // Erase the error source. +//! .map_src(drop) +//! }).join().unwrap(); +//! ``` +//! +//! Alternatively, use `.to_string()` to eagerly convert the error into a +//! human-readable message; e.g.: +//! +//! ``` +//! use zerocopy::*; +//! +//! let result: Result = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source) +//! // Eagerly render the error message. +//! .map_err(|err| err.to_string()) +//! }).join().unwrap(); +//! ``` +#[cfg(not(no_zerocopy_core_error_1_81_0))] +use core::error::Error; +use core::{ + convert::Infallible, + fmt::{self, Debug, Write}, + ops::Deref, +}; +#[cfg(all(no_zerocopy_core_error_1_81_0, any(feature = "std", test)))] +use std::error::Error; + +use crate::{util::SendSyncPhantomData, KnownLayout, TryFromBytes, Unaligned}; +#[cfg(doc)] +use crate::{FromBytes, Ref}; + +/// Zerocopy's generic error type. +/// +/// Generally speaking, zerocopy's conversions may fail for one of up to three +/// reasons: +/// - [`AlignmentError`]: the conversion source was improperly aligned +/// - [`SizeError`]: the conversion source was of incorrect size +/// - [`ValidityError`]: the conversion source contained invalid data +/// +/// However, not all conversions produce all errors. For instance, +/// [`FromBytes::ref_from_bytes`] may fail due to alignment or size issues, but +/// not validity issues. This generic error type captures these +/// (im)possibilities via parameterization: `A` is parameterized with +/// [`AlignmentError`], `S` is parameterized with [`SizeError`], and `V` is +/// parameterized with [`Infallible`]. +/// +/// Zerocopy never uses this type directly in its API. Rather, we provide three +/// pre-parameterized aliases: +/// - [`CastError`]: the error type of reference conversions +/// - [`TryCastError`]: the error type of fallible reference conversions +/// - [`TryReadError`]: the error type of fallible read conversions +#[derive(PartialEq, Eq, Clone)] +pub enum ConvertError { + /// The conversion source was improperly aligned. + Alignment(A), + /// The conversion source was of incorrect size. + Size(S), + /// The conversion source contained invalid data. + Validity(V), +} + +impl From, S, V>> + for ConvertError +{ + /// Infallibly discards the alignment error from this `ConvertError` since + /// `Dst` is unaligned. + /// + /// Since [`Dst: Unaligned`], it is impossible to encounter an alignment + /// error. This method permits discarding that alignment error infallibly + /// and replacing it with [`Infallible`]. + /// + /// [`Dst: Unaligned`]: crate::Unaligned + /// + /// # Examples + /// + /// ``` + /// use core::convert::Infallible; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, KnownLayout, Unaligned, Immutable)] + /// #[repr(C, packed)] + /// struct Bools { + /// one: bool, + /// two: bool, + /// many: [bool], + /// } + /// + /// impl Bools { + /// fn parse(bytes: &[u8]) -> Result<&Bools, AlignedTryCastError<&[u8], Bools>> { + /// // Since `Bools: Unaligned`, we can infallibly discard + /// // the alignment error. + /// Bools::try_ref_from_bytes(bytes).map_err(Into::into) + /// } + /// } + /// ``` + #[inline] + fn from(err: ConvertError, S, V>) -> ConvertError { + match err { + ConvertError::Alignment(e) => { + #[allow(unreachable_code)] + return ConvertError::Alignment(Infallible::from(e)); + } + ConvertError::Size(e) => ConvertError::Size(e), + ConvertError::Validity(e) => ConvertError::Validity(e), + } + } +} + +impl fmt::Debug for ConvertError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Alignment(e) => f.debug_tuple("Alignment").field(e).finish(), + Self::Size(e) => f.debug_tuple("Size").field(e).finish(), + Self::Validity(e) => f.debug_tuple("Validity").field(e).finish(), + } + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl fmt::Display for ConvertError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Alignment(e) => e.fmt(f), + Self::Size(e) => e.fmt(f), + Self::Validity(e) => e.fmt(f), + } + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl Error for ConvertError +where + A: fmt::Display + fmt::Debug, + S: fmt::Display + fmt::Debug, + V: fmt::Display + fmt::Debug, +{ +} + +/// The error emitted if the conversion source is improperly aligned. +pub struct AlignmentError { + /// The source value involved in the conversion. + src: Src, + /// The inner destination type involved in the conversion. + /// + /// INVARIANT: An `AlignmentError` may only be constructed if `Dst`'s + /// alignment requirement is greater than one. + _dst: SendSyncPhantomData, +} + +impl AlignmentError { + /// # Safety + /// + /// The caller must ensure that `Dst`'s alignment requirement is greater + /// than one. + pub(crate) unsafe fn new_unchecked(src: Src) -> Self { + // INVARIANT: The caller guarantees that `Dst`'s alignment requirement + // is greater than one. + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + pub(crate) fn with_src(self, new_src: NewSrc) -> AlignmentError { + // INVARIANT: `with_src` doesn't change the type of `Dst`, so the + // invariant that `Dst`'s alignment requirement is greater than one is + // preserved. + AlignmentError { src: new_src, _dst: SendSyncPhantomData::default() } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let unaligned = Unalign::new(0u16); + /// + /// // Attempt to deref `unaligned`. This might fail with an alignment error. + /// let maybe_n: Result<&u16, AlignmentError<&Unalign, u16>> = unaligned.try_deref(); + /// + /// // Map the error's source to its address as a usize. + /// let maybe_n: Result<&u16, AlignmentError> = maybe_n.map_err(|err| { + /// err.map_src(|src| src as *const _ as usize) + /// }); + /// ``` + #[inline] + pub fn map_src(self, f: impl FnOnce(Src) -> NewSrc) -> AlignmentError { + AlignmentError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + pub(crate) fn into(self) -> ConvertError { + ConvertError::Alignment(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Src: Deref, + Dst: KnownLayout, + { + #[allow(clippy::as_conversions)] + let addr = self.src.deref() as *const _ as *const (); + let addr_align = 2usize.pow((crate::util::AsAddress::addr(addr)).trailing_zeros()); + + f.write_str("\n\nSource type: ")?; + f.write_str(core::any::type_name::())?; + + f.write_str("\nSource address: ")?; + addr.fmt(f)?; + f.write_str(" (a multiple of ")?; + addr_align.fmt(f)?; + f.write_str(")")?; + + f.write_str("\nDestination type: ")?; + f.write_str(core::any::type_name::())?; + + f.write_str("\nDestination alignment: ")?; + ::LAYOUT.align.get().fmt(f)?; + + Ok(()) + } +} + +impl Clone for AlignmentError { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +impl PartialEq for AlignmentError { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl Eq for AlignmentError {} + +impl From> for Infallible { + #[inline(always)] + fn from(_: AlignmentError) -> Infallible { + // SAFETY: `AlignmentError`s can only be constructed when `Dst`'s + // alignment requirement is greater than one. In this block, `Dst: + // Unaligned`, which means that its alignment requirement is equal to + // one. Thus, it's not possible to reach here at runtime. + unsafe { core::hint::unreachable_unchecked() } + } +} + +#[cfg(test)] +impl AlignmentError { + // A convenience constructor so that test code doesn't need to write + // `unsafe`. + fn new_checked(src: Src) -> AlignmentError { + assert_ne!(core::mem::align_of::(), 1); + // SAFETY: The preceding assertion guarantees that `Dst`'s alignment + // requirement is greater than one. + unsafe { AlignmentError::new_unchecked(src) } + } +} + +impl fmt::Debug for AlignmentError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AlignmentError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl fmt::Display for AlignmentError +where + Src: Deref, + Dst: KnownLayout, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.")?; + + if cfg!(debug_assertions) { + self.display_verbose_extras(f) + } else { + Ok(()) + } + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl Error for AlignmentError +where + Src: Deref, + Dst: KnownLayout, +{ +} + +impl From> + for ConvertError, S, V> +{ + #[inline(always)] + fn from(err: AlignmentError) -> Self { + Self::Alignment(err) + } +} + +/// The error emitted if the conversion source is of incorrect size. +pub struct SizeError { + /// The source value involved in the conversion. + src: Src, + /// The inner destination type involved in the conversion. + _dst: SendSyncPhantomData, +} + +impl SizeError { + pub(crate) fn new(src: Src) -> Self { + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + /// Sets the source value associated with the conversion error. + pub(crate) fn with_src(self, new_src: NewSrc) -> SizeError { + SizeError { src: new_src, _dst: SendSyncPhantomData::default() } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 1, 2]; + /// + /// // Try to read a `u32` from `source`. This will fail because there are insufficient + /// // bytes in `source`. + /// let maybe_u32: Result> = u32::read_from_bytes(&source[..]); + /// + /// // Map the error's source to its size. + /// let maybe_u32: Result> = maybe_u32.map_err(|err| { + /// err.map_src(|src| src.len()) + /// }); + /// ``` + #[inline] + pub fn map_src(self, f: impl FnOnce(Src) -> NewSrc) -> SizeError { + SizeError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + /// Sets the destination type associated with the conversion error. + pub(crate) fn with_dst(self) -> SizeError { + SizeError { src: self.src, _dst: SendSyncPhantomData::default() } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into(self) -> ConvertError { + ConvertError::Size(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Src: Deref, + Dst: KnownLayout, + { + // include the source type + f.write_str("\nSource type: ")?; + f.write_str(core::any::type_name::())?; + + // include the source.deref() size + let src_size = core::mem::size_of_val(&*self.src); + f.write_str("\nSource size: ")?; + src_size.fmt(f)?; + f.write_str(" byte")?; + if src_size != 1 { + f.write_char('s')?; + } + + // if `Dst` is `Sized`, include the `Dst` size + if let crate::SizeInfo::Sized { size } = Dst::LAYOUT.size_info { + f.write_str("\nDestination size: ")?; + size.fmt(f)?; + f.write_str(" byte")?; + if size != 1 { + f.write_char('s')?; + } + } + + // include the destination type + f.write_str("\nDestination type: ")?; + f.write_str(core::any::type_name::())?; + + Ok(()) + } +} + +impl Clone for SizeError { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +impl PartialEq for SizeError { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl Eq for SizeError {} + +impl fmt::Debug for SizeError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SizeError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl fmt::Display for SizeError +where + Src: Deref, + Dst: KnownLayout, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.")?; + if cfg!(debug_assertions) { + f.write_str("\n")?; + self.display_verbose_extras(f)?; + } + Ok(()) + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl Error for SizeError +where + Src: Deref, + Dst: KnownLayout, +{ +} + +impl From> for ConvertError, V> { + #[inline(always)] + fn from(err: SizeError) -> Self { + Self::Size(err) + } +} + +/// The error emitted if the conversion source contains invalid data. +pub struct ValidityError { + /// The source value involved in the conversion. + pub(crate) src: Src, + /// The inner destination type involved in the conversion. + _dst: SendSyncPhantomData, +} + +impl ValidityError { + pub(crate) fn new(src: Src) -> Self { + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: u8 = 42; + /// + /// // Try to transmute the `source` to a `bool`. This will fail. + /// let maybe_bool: Result> = try_transmute!(source); + /// + /// // Drop the error's source. + /// let maybe_bool: Result> = maybe_bool.map_err(|err| { + /// err.map_src(drop) + /// }); + /// ``` + #[inline] + pub fn map_src(self, f: impl FnOnce(Src) -> NewSrc) -> ValidityError { + ValidityError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into(self) -> ConvertError { + ConvertError::Validity(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Dst: KnownLayout, + { + f.write_str("Destination type: ")?; + f.write_str(core::any::type_name::())?; + Ok(()) + } +} + +impl Clone for ValidityError { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +// SAFETY: `ValidityError` contains a single `Self::Inner = Src`, and no other +// non-ZST fields. `map` passes ownership of `self`'s sole `Self::Inner` to `f`. +unsafe impl crate::pointer::TryWithError + for crate::ValidityError +where + Dst: TryFromBytes + ?Sized, +{ + type Inner = Src; + type Mapped = crate::ValidityError; + #[inline] + fn map NewSrc>(self, f: F) -> Self::Mapped { + self.map_src(f) + } +} + +impl PartialEq for ValidityError { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl Eq for ValidityError {} + +impl fmt::Debug for ValidityError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ValidityError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl fmt::Display for ValidityError +where + Dst: KnownLayout + TryFromBytes, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the source bytes are not a valid value of the destination type.")?; + if cfg!(debug_assertions) { + f.write_str("\n\n")?; + self.display_verbose_extras(f)?; + } + Ok(()) + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl Error for ValidityError where Dst: KnownLayout + TryFromBytes {} + +impl From> + for ConvertError> +{ + #[inline(always)] + fn from(err: ValidityError) -> Self { + Self::Validity(err) + } +} + +/// The error type of reference conversions. +/// +/// Reference conversions, like [`FromBytes::ref_from_bytes`] may emit +/// [alignment](AlignmentError) and [size](SizeError) errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type CastError = + ConvertError, SizeError, Infallible>; + +impl CastError { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(e) => e.src, + Self::Size(e) => e.src, + Self::Validity(i) => match i {}, + } + } + + /// Sets the source value associated with the conversion error. + pub(crate) fn with_src(self, new_src: NewSrc) -> CastError { + match self { + Self::Alignment(e) => CastError::Alignment(e.with_src(new_src)), + Self::Size(e) => CastError::Size(e.with_src(new_src)), + Self::Validity(i) => match i {}, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 1, 2]; + /// + /// // Try to read a `u32` from `source`. This will fail because there are insufficient + /// // bytes in `source`. + /// let maybe_u32: Result<&u32, CastError<&[u8], u32>> = u32::ref_from_bytes(&source[..]); + /// + /// // Map the error's source to its size and address. + /// let maybe_u32: Result<&u32, CastError<(usize, usize), u32>> = maybe_u32.map_err(|err| { + /// err.map_src(|src| (src.len(), src.as_ptr() as usize)) + /// }); + /// ``` + #[inline] + pub fn map_src(self, f: impl FnOnce(Src) -> NewSrc) -> CastError { + match self { + Self::Alignment(e) => CastError::Alignment(e.map_src(f)), + Self::Size(e) => CastError::Size(e.map_src(f)), + Self::Validity(i) => match i {}, + } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into(self) -> TryCastError + where + Dst: TryFromBytes, + { + match self { + Self::Alignment(e) => TryCastError::Alignment(e), + Self::Size(e) => TryCastError::Size(e), + Self::Validity(i) => match i {}, + } + } +} + +// SAFETY: `CastError` is either a single `AlignmentError` or a single +// `SizeError`. In either case, it contains a single `Self::Inner = Src`, and no +// other non-ZST fields. `map` passes ownership of `self`'s sole `Self::Inner` +// to `f`. +unsafe impl crate::pointer::TryWithError for crate::CastError +where + Dst: ?Sized, +{ + type Inner = Src; + type Mapped = crate::CastError; + + #[inline] + fn map NewSrc>(self, f: F) -> Self::Mapped { + self.map_src(f) + } +} + +impl From> for SizeError { + /// Infallibly extracts the [`SizeError`] from this `CastError` since `Dst` + /// is unaligned. + /// + /// Since [`Dst: Unaligned`], it is impossible to encounter an alignment + /// error, and so the only error that can be encountered at runtime is a + /// [`SizeError`]. This method permits extracting that `SizeError` + /// infallibly. + /// + /// [`Dst: Unaligned`]: crate::Unaligned + /// + /// # Examples + /// + /// ```rust + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct UdpHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C, packed)] + /// struct UdpPacket { + /// header: UdpHeader, + /// body: [u8], + /// } + /// + /// impl UdpPacket { + /// pub fn parse(bytes: &[u8]) -> Result<&UdpPacket, SizeError<&[u8], UdpPacket>> { + /// // Since `UdpPacket: Unaligned`, we can map the `CastError` to a `SizeError`. + /// UdpPacket::ref_from_bytes(bytes).map_err(Into::into) + /// } + /// } + /// ``` + #[inline(always)] + fn from(err: CastError) -> SizeError { + match err { + #[allow(unreachable_code)] + CastError::Alignment(e) => match Infallible::from(e) {}, + CastError::Size(e) => e, + CastError::Validity(i) => match i {}, + } + } +} + +/// The error type of fallible reference conversions. +/// +/// Fallible reference conversions, like [`TryFromBytes::try_ref_from_bytes`] +/// may emit [alignment](AlignmentError), [size](SizeError), and +/// [validity](ValidityError) errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type TryCastError = + ConvertError, SizeError, ValidityError>; + +// FIXME(#1139): Remove the `TryFromBytes` here and in other downstream +// locations (all the way to `ValidityError`) if we determine it's not necessary +// for rich validity errors. +impl TryCastError { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(e) => e.src, + Self::Size(e) => e.src, + Self::Validity(e) => e.src, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use core::num::NonZeroU32; + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 0, 0]; + /// + /// // Try to read a `NonZeroU32` from `source`. + /// let maybe_u32: Result<&NonZeroU32, TryCastError<&[u8], NonZeroU32>> + /// = NonZeroU32::try_ref_from_bytes(&source[..]); + /// + /// // Map the error's source to its size and address. + /// let maybe_u32: Result<&NonZeroU32, TryCastError<(usize, usize), NonZeroU32>> = + /// maybe_u32.map_err(|err| { + /// err.map_src(|src| (src.len(), src.as_ptr() as usize)) + /// }); + /// ``` + #[inline] + pub fn map_src(self, f: impl FnOnce(Src) -> NewSrc) -> TryCastError { + match self { + Self::Alignment(e) => TryCastError::Alignment(e.map_src(f)), + Self::Size(e) => TryCastError::Size(e.map_src(f)), + Self::Validity(e) => TryCastError::Validity(e.map_src(f)), + } + } +} + +impl From> for TryCastError { + #[inline] + fn from(value: CastError) -> Self { + match value { + CastError::Alignment(e) => Self::Alignment(e), + CastError::Size(e) => Self::Size(e), + CastError::Validity(i) => match i {}, + } + } +} + +/// The error type of fallible read-conversions. +/// +/// Fallible read-conversions, like [`TryFromBytes::try_read_from_bytes`] may +/// emit [size](SizeError) and [validity](ValidityError) errors, but not +/// alignment errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type TryReadError = + ConvertError, ValidityError>; + +impl TryReadError { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(i) => match i {}, + Self::Size(e) => e.src, + Self::Validity(e) => e.src, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use core::num::NonZeroU32; + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 0, 0]; + /// + /// // Try to read a `NonZeroU32` from `source`. + /// let maybe_u32: Result> + /// = NonZeroU32::try_read_from_bytes(&source[..]); + /// + /// // Map the error's source to its size. + /// let maybe_u32: Result> = + /// maybe_u32.map_err(|err| { + /// err.map_src(|src| src.len()) + /// }); + /// ``` + #[inline] + pub fn map_src(self, f: impl FnOnce(Src) -> NewSrc) -> TryReadError { + match self { + Self::Alignment(i) => match i {}, + Self::Size(e) => TryReadError::Size(e.map_src(f)), + Self::Validity(e) => TryReadError::Validity(e.map_src(f)), + } + } +} + +/// The error type of well-aligned, fallible casts. +/// +/// This is like [`TryCastError`], but for casts that are always well-aligned. +/// It is identical to `TryCastError`, except that its alignment error is +/// [`Infallible`]. +/// +/// As of this writing, none of zerocopy's API produces this error directly. +/// However, it is useful since it permits users to infallibly discard alignment +/// errors when they can prove statically that alignment errors are impossible. +/// +/// # Examples +/// +/// ``` +/// use core::convert::Infallible; +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(TryFromBytes, KnownLayout, Unaligned, Immutable)] +/// #[repr(C, packed)] +/// struct Bools { +/// one: bool, +/// two: bool, +/// many: [bool], +/// } +/// +/// impl Bools { +/// fn parse(bytes: &[u8]) -> Result<&Bools, AlignedTryCastError<&[u8], Bools>> { +/// // Since `Bools: Unaligned`, we can infallibly discard +/// // the alignment error. +/// Bools::try_ref_from_bytes(bytes).map_err(Into::into) +/// } +/// } +/// ``` +#[allow(type_alias_bounds)] +pub type AlignedTryCastError = + ConvertError, ValidityError>; + +/// The error type of a failed allocation. +/// +/// This type is intended to be deprecated in favor of the standard library's +/// [`AllocError`] type once it is stabilized. When that happens, this type will +/// be replaced by a type alias to the standard library type. We do not intend +/// to treat this as a breaking change; users who wish to avoid breakage should +/// avoid writing code which assumes that this is *not* such an alias. For +/// example, implementing the same trait for both types will result in an impl +/// conflict once this type is an alias. +/// +/// [`AllocError`]: https://doc.rust-lang.org/alloc/alloc/struct.AllocError.html +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct AllocError; + +#[cfg(test)] +mod tests { + use core::convert::Infallible; + + use super::*; + + #[test] + fn test_send_sync() { + // Test that all error types are `Send + Sync` even if `Dst: !Send + + // !Sync`. + + #[allow(dead_code)] + fn is_send_sync(_t: T) {} + + #[allow(dead_code)] + fn alignment_err_is_send_sync(err: AlignmentError) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn size_err_is_send_sync(err: SizeError) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn validity_err_is_send_sync( + err: ValidityError, + ) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn convert_error_is_send_sync( + err: ConvertError< + AlignmentError, + SizeError, + ValidityError, + >, + ) { + is_send_sync(err) + } + } + + #[test] + fn test_eq_partial_eq_clone() { + // Test that all error types implement `Eq`, `PartialEq` + // and `Clone` if src does + // even if `Dst: !Eq`, `!PartialEq`, `!Clone`. + + #[allow(dead_code)] + fn is_eq_partial_eq_clone(_t: T) {} + + #[allow(dead_code)] + fn alignment_err_is_eq_partial_eq_clone( + err: AlignmentError, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn size_err_is_eq_partial_eq_clone( + err: SizeError, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn validity_err_is_eq_partial_eq_clone( + err: ValidityError, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn convert_error_is_eq_partial_eq_clone( + err: ConvertError< + AlignmentError, + SizeError, + ValidityError, + >, + ) { + is_eq_partial_eq_clone(err) + } + } + + #[test] + fn alignment_display() { + #[repr(C, align(128))] + struct Aligned { + bytes: [u8; 128], + } + + impl_known_layout!(elain::Align::<8>); + + let aligned = Aligned { bytes: [0; 128] }; + + let bytes = &aligned.bytes[1..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 1)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[2..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 2)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[3..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 1)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[4..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 4)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + } + + #[test] + fn size_display() { + assert_eq!( + SizeError::<_, [u8]>::new(&[0u8; 2][..]).to_string(), + "The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.\n\ + \nSource type: &[u8]\ + \nSource size: 2 bytes\ + \nDestination type: [u8]" + ); + + assert_eq!( + SizeError::<_, [u8; 2]>::new(&[0u8; 1][..]).to_string(), + "The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.\n\ + \nSource type: &[u8]\ + \nSource size: 1 byte\ + \nDestination size: 2 bytes\ + \nDestination type: [u8; 2]" + ); + } + + #[test] + fn validity_display() { + assert_eq!( + ValidityError::<_, bool>::new(&[2u8; 1][..]).to_string(), + "The conversion failed because the source bytes are not a valid value of the destination type.\n\ + \n\ + Destination type: bool" + ); + } + + #[test] + fn test_convert_error_debug() { + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Alignment(AlignmentError::new_checked(&[0u8])); + assert_eq!(format!("{:?}", err), "Alignment(AlignmentError)"); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + assert_eq!(format!("{:?}", err), "Size(SizeError)"); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + assert_eq!(format!("{:?}", err), "Validity(ValidityError)"); + } + + #[test] + fn test_convert_error_from_unaligned() { + // u8 is Unaligned + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + let converted: ConvertError, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Size(_) => {} + _ => panic!("Expected Size error"), + } + } + + #[test] + fn test_alignment_error_display_debug() { + let err: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[0u8]); + assert!(format!("{:?}", err).contains("AlignmentError")); + assert!(format!("{}", err).contains("address of the source is not a multiple")); + } + + #[test] + fn test_size_error_display_debug() { + let err: SizeError<&[u8], u16> = SizeError::new(&[0u8]); + assert!(format!("{:?}", err).contains("SizeError")); + assert!(format!("{}", err).contains("source was incorrectly sized")); + } + + #[test] + fn test_validity_error_display_debug() { + let err: ValidityError<&[u8], bool> = ValidityError::new(&[0u8]); + assert!(format!("{:?}", err).contains("ValidityError")); + assert!(format!("{}", err).contains("source bytes are not a valid value")); + } + + #[test] + fn test_convert_error_display_debug_more() { + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Alignment(AlignmentError::new_checked(&[0u8])); + assert!(format!("{}", err).contains("address of the source is not a multiple")); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + assert!(format!("{}", err).contains("source was incorrectly sized")); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + assert!(format!("{}", err).contains("source bytes are not a valid value")); + } + + #[test] + fn test_alignment_error_methods() { + let err: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[0u8]); + + // into_src + let src = err.clone().into_src(); + assert_eq!(src, &[0u8]); + + // into + let converted: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = err.clone().into(); + match converted { + ConvertError::Alignment(_) => {} + _ => panic!("Expected Alignment error"), + } + + // clone + let cloned = err.clone(); + assert_eq!(err, cloned); + + // eq + assert_eq!(err, cloned); + let err2: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[1u8]); + assert_ne!(err, err2); + } + + #[test] + fn test_convert_error_from_unaligned_variants() { + // u8 is Unaligned + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + let converted: ConvertError, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Validity(_) => {} + _ => panic!("Expected Validity error"), + } + + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + let converted: ConvertError, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Size(_) => {} + _ => panic!("Expected Size error"), + } + } +} diff --git a/rust/zerocopy/src/impls.rs b/rust/zerocopy/src/impls.rs new file mode 100644 index 000000000000..80538bfc8a26 --- /dev/null +++ b/rust/zerocopy/src/impls.rs @@ -0,0 +1,2387 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{ + cell::{Cell, UnsafeCell}, + mem::MaybeUninit as CoreMaybeUninit, + ptr::NonNull, +}; + +use super::*; +use crate::pointer::cast::{CastSizedExact, CastUnsized}; + +// SAFETY: Per the reference [1], "the unit tuple (`()`) ... is guaranteed as a +// zero-sized type to have a size of 0 and an alignment of 1." +// - `Immutable`: `()` self-evidently does not contain any `UnsafeCell`s. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: There is only +// one possible sequence of 0 bytes, and `()` is inhabited. +// - `IntoBytes`: Since `()` has size 0, it contains no padding bytes. +// - `Unaligned`: `()` has alignment 1. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#tuple-layout +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!((): Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(()); +}; + +// SAFETY: +// - `Immutable`: These types self-evidently do not contain any `UnsafeCell`s. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: all bit +// patterns are valid for numeric types [1] +// - `IntoBytes`: numeric types have no padding bytes [1] +// - `Unaligned` (`u8` and `i8` only): The reference [2] specifies the size of +// `u8` and `i8` as 1 byte. We also know that: +// - Alignment is >= 1 [3] +// - Size is an integer multiple of alignment [4] +// - The only value >= 1 for which 1 is an integer multiple is 1 Therefore, +// the only possible alignment for `u8` and `i8` is 1. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/types/numeric.html#bit-validity: +// +// For every numeric type, `T`, the bit validity of `T` is equivalent to +// the bit validity of `[u8; size_of::()]`. An uninitialized byte is +// not a valid `u8`. +// +// [2] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#primitive-data-layout +// +// [3] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: +// +// Alignment is measured in bytes, and must be at least 1. +// +// [4] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: +// +// The size of a value is always a multiple of its alignment. +// +// FIXME(#278): Once we've updated the trait docs to refer to `u8`s rather than +// bits or bytes, update this comment, especially the reference to [1]. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(u8: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + unsafe_impl!(i8: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(u8, i8); + unsafe_impl!(u16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(usize: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(isize: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(f32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(f64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + #[cfg(feature = "float-nightly")] + unsafe_impl!(#[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] f16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + #[cfg(feature = "float-nightly")] + unsafe_impl!(#[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] f128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); +}; + +// SAFETY: +// - `Immutable`: `bool` self-evidently does not contain any `UnsafeCell`s. +// - `FromZeros`: Valid since "[t]he value false has the bit pattern 0x00" [1]. +// - `IntoBytes`: Since "the boolean type has a size and alignment of 1 each" +// and "The value false has the bit pattern 0x00 and the value true has the +// bit pattern 0x01" [1]. Thus, the only byte of the bool is always +// initialized. +// - `Unaligned`: Per the reference [1], "[a]n object with the boolean type has +// a size and alignment of 1 each." +// +// [1] https://doc.rust-lang.org/1.81.0/reference/types/boolean.html +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(bool: Immutable, FromZeros, IntoBytes, Unaligned) }; +assert_unaligned!(bool); + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe` refers to a valid `bool`. We only return true if the `u8` value +// is 0 or 1, and both of these are valid values for `bool` [1]. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/types/boolean.html: +// +// The value false has the bit pattern 0x00 and the value true has the bit +// pattern 0x01. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for bool; |byte| { + let byte = byte.transmute_with::(); + *byte.unaligned_as_ref() < 2 + }) +}; + +// SAFETY: +// - `Immutable`: `char` self-evidently does not contain any `UnsafeCell`s. +// - `FromZeros`: Per reference [1], "[a] value of type char is a Unicode scalar +// value (i.e. a code point that is not a surrogate), represented as a 32-bit +// unsigned word in the 0x0000 to 0xD7FF or 0xE000 to 0x10FFFF range" which +// contains 0x0000. +// - `IntoBytes`: `char` is per reference [1] "represented as a 32-bit unsigned +// word" (`u32`) which is `IntoBytes`. Note that unlike `u32`, not all bit +// patterns are valid for `char`. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/types/textual.html +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(char: Immutable, FromZeros, IntoBytes) }; + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe` refers to a valid `char`. `char::from_u32` guarantees that it +// returns `None` if its input is not a valid `char` [1]. +// +// [1] Per https://doc.rust-lang.org/core/primitive.char.html#method.from_u32: +// +// `from_u32()` will return `None` if the input is not a valid value for a +// `char`. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for char; |c| { + let c = c.transmute_with::, invariant::Valid, CastSizedExact, BecauseImmutable>(); + let c = c.read().into_inner(); + char::from_u32(c).is_some() + }); +}; + +// SAFETY: Per the Reference [1], `str` has the same layout as `[u8]`. +// - `Immutable`: `[u8]` does not contain any `UnsafeCell`s. +// - `FromZeros`, `IntoBytes`, `Unaligned`: `[u8]` is `FromZeros`, `IntoBytes`, +// and `Unaligned`. +// +// Note that we don't `assert_unaligned!(str)` because `assert_unaligned!` uses +// `align_of`, which only works for `Sized` types. +// +// FIXME(#429): Improve safety proof for `FromZeros` and `IntoBytes`; having the same +// layout as `[u8]` isn't sufficient. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#str-layout: +// +// String slices are a UTF-8 representation of characters that have the same +// layout as slices of type `[u8]`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(str: Immutable, FromZeros, IntoBytes, Unaligned) }; + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe` refers to a valid `str`. `str::from_utf8` guarantees that it +// returns `Err` if its input is not a valid `str` [1]. +// +// [1] Per https://doc.rust-lang.org/core/str/fn.from_utf8.html#errors: +// +// Returns `Err` if the slice is not UTF-8. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for str; |c| { + let c = c.transmute_with::<[u8], invariant::Valid, CastUnsized, BecauseImmutable>(); + let c = c.unaligned_as_ref(); + core::str::from_utf8(c).is_ok() + }) +}; + +macro_rules! unsafe_impl_try_from_bytes_for_nonzero { + ($($nonzero:ident[$prim:ty]),*) => { + $( + unsafe_impl!(=> TryFromBytes for $nonzero; |n| { + let n = n.transmute_with::, invariant::Valid, CastSizedExact, BecauseImmutable>(); + $nonzero::new(n.read().into_inner()).is_some() + }); + )* + } +} + +// `NonZeroXxx` is `IntoBytes`, but not `FromZeros` or `FromBytes`. +// +// SAFETY: +// - `IntoBytes`: `NonZeroXxx` has the same layout as its associated primitive. +// Since it is the same size, this guarantees it has no padding - integers +// have no padding, and there's no room for padding if it can represent all +// of the same values except 0. +// - `Unaligned`: `NonZeroU8` and `NonZeroI8` document that `Option` +// and `Option` both have size 1. [1] [2] This is worded in a way +// that makes it unclear whether it's meant as a guarantee, but given the +// purpose of those types, it's virtually unthinkable that that would ever +// change. `Option` cannot be smaller than its contained type, which implies +// that, and `NonZeroX8` are of size 1 or 0. `NonZeroX8` can represent +// multiple states, so they cannot be 0 bytes, which means that they must be 1 +// byte. The only valid alignment for a 1-byte type is 1. +// +// FIXME(#429): +// - Add quotes from documentation. +// - Add safety comment for `Immutable`. How can we prove that `NonZeroXxx` +// doesn't contain any `UnsafeCell`s? It's obviously true, but it's not clear +// how we'd prove it short of adding text to the stdlib docs that says so +// explicitly, which likely wouldn't be accepted. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroU8.html: +// +// `NonZeroU8` is guaranteed to have the same layout and bit validity as `u8` with +// the exception that 0 is not a valid instance. +// +// [2] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroI8.html: +// +// `NonZeroI8` is guaranteed to have the same layout and bit validity as `i8` with +// the exception that 0 is not a valid instance. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(NonZeroU8: Immutable, IntoBytes, Unaligned); + unsafe_impl!(NonZeroI8: Immutable, IntoBytes, Unaligned); + assert_unaligned!(NonZeroU8, NonZeroI8); + unsafe_impl!(NonZeroU16: Immutable, IntoBytes); + unsafe_impl!(NonZeroI16: Immutable, IntoBytes); + unsafe_impl!(NonZeroU32: Immutable, IntoBytes); + unsafe_impl!(NonZeroI32: Immutable, IntoBytes); + unsafe_impl!(NonZeroU64: Immutable, IntoBytes); + unsafe_impl!(NonZeroI64: Immutable, IntoBytes); + unsafe_impl!(NonZeroU128: Immutable, IntoBytes); + unsafe_impl!(NonZeroI128: Immutable, IntoBytes); + unsafe_impl!(NonZeroUsize: Immutable, IntoBytes); + unsafe_impl!(NonZeroIsize: Immutable, IntoBytes); + unsafe_impl_try_from_bytes_for_nonzero!( + NonZeroU8[u8], + NonZeroI8[i8], + NonZeroU16[u16], + NonZeroI16[i16], + NonZeroU32[u32], + NonZeroI32[i32], + NonZeroU64[u64], + NonZeroI64[i64], + NonZeroU128[u128], + NonZeroI128[i128], + NonZeroUsize[usize], + NonZeroIsize[isize] + ); +}; + +// SAFETY: +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`, `IntoBytes`: +// The Rust compiler reuses `0` value to represent `None`, so +// `size_of::>() == size_of::()`; see `NonZeroXxx` +// documentation. +// - `Unaligned`: `NonZeroU8` and `NonZeroI8` document that `Option` +// and `Option` both have size 1. [1] [2] This is worded in a way +// that makes it unclear whether it's meant as a guarantee, but given the +// purpose of those types, it's virtually unthinkable that that would ever +// change. The only valid alignment for a 1-byte type is 1. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroU8.html: +// +// `Option` is guaranteed to be compatible with `u8`, including in FFI. +// +// Thanks to the null pointer optimization, `NonZeroU8` and `Option` +// are guaranteed to have the same size and alignment: +// +// [2] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroI8.html: +// +// `Option` is guaranteed to be compatible with `i8`, including in FFI. +// +// Thanks to the null pointer optimization, `NonZeroI8` and `Option` +// are guaranteed to have the same size and alignment: +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(Option, Option); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option: TryFromBytes, FromZeros, FromBytes, IntoBytes); +}; + +// SAFETY: While it's not fully documented, the consensus is that `Box` does +// not contain any `UnsafeCell`s for `T: Sized` [1]. This is not a complete +// proof, but we are accepting this as a known risk per #1358. +// +// [1] https://github.com/rust-lang/unsafe-code-guidelines/issues/492 +#[cfg(feature = "alloc")] +const _: () = unsafe { + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T: Sized => Immutable for Box + ) +}; + +// SAFETY: The following types can be transmuted from `[0u8; size_of::()]`. [1] +// +// [1] Per https://doc.rust-lang.org/1.89.0/core/option/index.html#representation: +// +// Rust guarantees to optimize the following types `T` such that [`Option`] +// has the same size and alignment as `T`. In some of these cases, Rust +// further guarantees that `transmute::<_, Option>([0u8; size_of::()])` +// is sound and produces `Option::::None`. These cases are identified by +// the second column: +// +// | `T` | `transmute::<_, Option>([0u8; size_of::()])` sound? | +// |-----------------------------------|-----------------------------------------------------------| +// | [`Box`] | when `U: Sized` | +// | `&U` | when `U: Sized` | +// | `&mut U` | when `U: Sized` | +// | [`ptr::NonNull`] | when `U: Sized` | +// | `fn`, `extern "C" fn`[^extern_fn] | always | +// +// [^extern_fn]: this remains true for `unsafe` variants, any argument/return +// types, and any other ABI: `[unsafe] extern "abi" fn` (_e.g._, `extern +// "system" fn`) +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + #[cfg(feature = "alloc")] + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T => TryFromBytes for Option>; |c| pointer::is_zeroed(c) + ); + #[cfg(feature = "alloc")] + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T => FromZeros for Option> + ); + unsafe_impl!( + T => TryFromBytes for Option<&'_ T>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option<&'_ T>); + unsafe_impl!( + T => TryFromBytes for Option<&'_ mut T>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option<&'_ mut T>); + unsafe_impl!( + T => TryFromBytes for Option>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option>); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_unsafe_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_unsafe_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_extern_c_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_extern_c_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_unsafe_extern_c_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_unsafe_extern_c_fn!(...); + |c| pointer::is_zeroed(c) + ); +}; + +// SAFETY: `[unsafe] [extern "C"] fn()` self-evidently do not contain +// `UnsafeCell`s. This is not a proof, but we are accepting this as a known risk +// per #1358. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_unsafe_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_extern_c_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_unsafe_extern_c_fn!(...)); +}; + +#[cfg(all( + not(no_zerocopy_target_has_atomics_1_60_0), + any( + target_has_atomic = "8", + target_has_atomic = "16", + target_has_atomic = "32", + target_has_atomic = "64", + target_has_atomic = "ptr" + ) +))] +#[cfg_attr(doc_cfg, doc(cfg(rust = "1.60.0")))] +mod atomics { + use super::*; + + macro_rules! impl_traits_for_atomics { + ($($atomics:tt [$primitives:ty]),* $(,)?) => { + $( + impl_known_layout!($atomics); + impl_for_transmute_from!(=> FromZeros for $atomics [$primitives]); + impl_for_transmute_from!(=> FromBytes for $atomics [$primitives]); + impl_for_transmute_from!(=> TryFromBytes for $atomics [$primitives]); + impl_for_transmute_from!(=> IntoBytes for $atomics [$primitives]); + )* + }; + } + + /// Implements `TransmuteFrom` for `$atomic`, `$prim`, and + /// `UnsafeCell<$prim>`. + /// + /// # Safety + /// + /// `$atomic` must have the same size and bit validity as `$prim`. + macro_rules! unsafe_impl_transmute_from_for_atomic { + ($($($tyvar:ident)? => $atomic:ty [$prim:ty]),*) => {{ + crate::util::macros::__unsafe(); + + use crate::pointer::{SizeEq, TransmuteFrom, invariant::Valid}; + + $( + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same size and bit validity. + unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Valid, Valid> for $prim {} + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same size and bit validity. + unsafe impl<$($tyvar)?> TransmuteFrom<$prim, Valid, Valid> for $atomic {} + + impl<$($tyvar)?> SizeEq> for ReadOnly<$prim> { + type CastFrom = $crate::pointer::cast::CastSizedExact; + } + + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same bit validity. `UnsafeCell` has the same bit + // validity as `T` [1]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell` has the same in-memory representation as + // its inner type `T`. A consequence of this guarantee is that + // it is possible to convert between `T` and `UnsafeCell`. + unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Valid, Valid> for core::cell::UnsafeCell<$prim> {} + // SAFETY: See previous safety comment. + unsafe impl<$($tyvar)?> TransmuteFrom, Valid, Valid> for $atomic {} + )* + }}; + } + + #[cfg(target_has_atomic = "8")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "8")))] + mod atomic_8 { + use core::sync::atomic::{AtomicBool, AtomicI8, AtomicU8}; + + use super::*; + + impl_traits_for_atomics!(AtomicU8[u8], AtomicI8[i8]); + + impl_known_layout!(AtomicBool); + impl_for_transmute_from!(=> FromZeros for AtomicBool [bool]); + impl_for_transmute_from!(=> TryFromBytes for AtomicBool [bool]); + impl_for_transmute_from!(=> IntoBytes for AtomicBool [bool]); + + // SAFETY: Per [1], `AtomicBool`, `AtomicU8`, and `AtomicI8` have the + // same size as `bool`, `u8`, and `i8` respectively. Since a type's + // alignment cannot be smaller than 1 [2], and since its alignment + // cannot be greater than its size [3], the only possible value for the + // alignment is 1. Thus, it is sound to implement `Unaligned`. + // + // [1] Per (for example) https://doc.rust-lang.org/1.81.0/std/sync/atomic/struct.AtomicU8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: + // + // Alignment is measured in bytes, and must be at least 1. + // + // [3] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: + // + // The size of a value is always a multiple of its alignment. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl!(AtomicBool: Unaligned); + unsafe_impl!(AtomicU8: Unaligned); + unsafe_impl!(AtomicI8: Unaligned); + assert_unaligned!(AtomicBool, AtomicU8, AtomicI8); + }; + + // SAFETY: `AtomicU8`, `AtomicI8`, and `AtomicBool` have the same size + // and bit validity as `u8`, `i8`, and `bool` respectively [1][2][3]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type, `u8`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type, `i8`. + // + // [3] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicBool.html: + // + // This type has the same size, alignment, and bit validity a `bool`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!( + => AtomicU8 [u8], + => AtomicI8 [i8], + => AtomicBool [bool] + ) + }; + } + + #[cfg(target_has_atomic = "16")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "16")))] + mod atomic_16 { + use core::sync::atomic::{AtomicI16, AtomicU16}; + + use super::*; + + impl_traits_for_atomics!(AtomicU16[u16], AtomicI16[i16]); + + // SAFETY: `AtomicU16` and `AtomicI16` have the same size and bit + // validity as `u16` and `i16` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU16.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u16`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI16.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i16`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU16 [u16], => AtomicI16 [i16]) + }; + } + + #[cfg(target_has_atomic = "32")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "32")))] + mod atomic_32 { + use core::sync::atomic::{AtomicI32, AtomicU32}; + + use super::*; + + impl_traits_for_atomics!(AtomicU32[u32], AtomicI32[i32]); + + // SAFETY: `AtomicU32` and `AtomicI32` have the same size and bit + // validity as `u32` and `i32` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU32.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u32`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI32.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i32`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU32 [u32], => AtomicI32 [i32]) + }; + } + + #[cfg(target_has_atomic = "64")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "64")))] + mod atomic_64 { + use core::sync::atomic::{AtomicI64, AtomicU64}; + + use super::*; + + impl_traits_for_atomics!(AtomicU64[u64], AtomicI64[i64]); + + // SAFETY: `AtomicU64` and `AtomicI64` have the same size and bit + // validity as `u64` and `i64` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU64.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u64`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI64.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i64`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU64 [u64], => AtomicI64 [i64]) + }; + } + + #[cfg(target_has_atomic = "ptr")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "ptr")))] + mod atomic_ptr { + use core::sync::atomic::{AtomicIsize, AtomicPtr, AtomicUsize}; + + use super::*; + + impl_traits_for_atomics!(AtomicUsize[usize], AtomicIsize[isize]); + + // FIXME(#170): Implement `FromBytes` and `IntoBytes` once we implement + // those traits for `*mut T`. + impl_known_layout!(T => AtomicPtr); + impl_for_transmute_from!(T => TryFromBytes for AtomicPtr [*mut T]); + impl_for_transmute_from!(T => FromZeros for AtomicPtr [*mut T]); + + // SAFETY: `AtomicUsize` and `AtomicIsize` have the same size and bit + // validity as `usize` and `isize` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicUsize.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `usize`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicIsize.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `isize`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicUsize [usize], => AtomicIsize [isize]) + }; + + // SAFETY: Per + // https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicPtr.html: + // + // This type has the same size and bit validity as a `*mut T`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { unsafe_impl_transmute_from_for_atomic!(T => AtomicPtr [*mut T]) }; + } +} + +// SAFETY: Per reference [1]: "For all T, the following are guaranteed: +// size_of::>() == 0 align_of::>() == 1". This +// gives: +// - `Immutable`: `PhantomData` has no fields. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: There is only +// one possible sequence of 0 bytes, and `PhantomData` is inhabited. +// - `IntoBytes`: Since `PhantomData` has size 0, it contains no padding bytes. +// - `Unaligned`: Per the preceding reference, `PhantomData` has alignment 1. +// +// [1] https://doc.rust-lang.org/1.81.0/std/marker/struct.PhantomData.html#layout-1 +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for PhantomData); + unsafe_impl!(T: ?Sized => TryFromBytes for PhantomData); + unsafe_impl!(T: ?Sized => FromZeros for PhantomData); + unsafe_impl!(T: ?Sized => FromBytes for PhantomData); + unsafe_impl!(T: ?Sized => IntoBytes for PhantomData); + unsafe_impl!(T: ?Sized => Unaligned for PhantomData); + assert_unaligned!(PhantomData<()>, PhantomData, PhantomData); +}; + +impl_for_transmute_from!(T: TryFromBytes => TryFromBytes for Wrapping[T]); +impl_for_transmute_from!(T: FromZeros => FromZeros for Wrapping[T]); +impl_for_transmute_from!(T: FromBytes => FromBytes for Wrapping[T]); +impl_for_transmute_from!(T: IntoBytes => IntoBytes for Wrapping[T]); +assert_unaligned!(Wrapping<()>, Wrapping); + +// SAFETY: Per [1], `Wrapping` has the same layout as `T`. Since its single +// field (of type `T`) is public, it would be a breaking change to add or remove +// fields. Thus, we know that `Wrapping` contains a `T` (as opposed to just +// having the same size and alignment as `T`) with no pre- or post-padding. +// Thus, `Wrapping` must have `UnsafeCell`s covering the same byte ranges as +// `Inner = T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/struct.Wrapping.html#layout-1: +// +// `Wrapping` is guaranteed to have the same layout and ABI as `T` +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for Wrapping) }; + +// SAFETY: Per [1] in the preceding safety comment, `Wrapping` has the same +// alignment as `T`. +const _: () = unsafe { unsafe_impl!(T: Unaligned => Unaligned for Wrapping) }; + +// SAFETY: `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: +// `MaybeUninit` has no restrictions on its contents. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T => TryFromBytes for CoreMaybeUninit); + unsafe_impl!(T => FromZeros for CoreMaybeUninit); + unsafe_impl!(T => FromBytes for CoreMaybeUninit); +}; + +// SAFETY: `MaybeUninit` has `UnsafeCell`s covering the same byte ranges as +// `Inner = T`. This is not explicitly documented, but it can be inferred. Per +// [1], `MaybeUninit` has the same size as `T`. Further, note the signature +// of `MaybeUninit::assume_init_ref` [2]: +// +// pub unsafe fn assume_init_ref(&self) -> &T +// +// If the argument `&MaybeUninit` and the returned `&T` had `UnsafeCell`s at +// different offsets, this would be unsound. Its existence is proof that this is +// not the case. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: +// +// `MaybeUninit` is guaranteed to have the same size, alignment, and ABI as +// `T`. +// +// [2] https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#method.assume_init_ref +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for CoreMaybeUninit) }; + +// SAFETY: Per [1] in the preceding safety comment, `MaybeUninit` has the +// same alignment as `T`. +const _: () = unsafe { unsafe_impl!(T: Unaligned => Unaligned for CoreMaybeUninit) }; +assert_unaligned!(CoreMaybeUninit<()>, CoreMaybeUninit); + +// SAFETY: `ManuallyDrop` has the same layout as `T` [1]. This strongly +// implies, but does not guarantee, that it contains `UnsafeCell`s covering the +// same byte ranges as in `T`. However, it also implements `Defer` +// [2], which provides the ability to convert `&ManuallyDrop -> &T`. This, +// combined with having the same size as `T`, implies that `ManuallyDrop` +// exactly contains a `T` with the same fields and `UnsafeCell`s covering the +// same byte ranges, or else the `Deref` impl would permit safe code to obtain +// different shared references to the same region of memory with different +// `UnsafeCell` coverage, which would in turn permit interior mutation that +// would violate the invariants of a shared reference. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop` is guaranteed to have the same layout and bit validity as +// `T` +// +// [2] https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html#impl-Deref-for-ManuallyDrop%3CT%3E +const _: () = unsafe { unsafe_impl!(T: ?Sized + Immutable => Immutable for ManuallyDrop) }; + +impl_for_transmute_from!(T: ?Sized + TryFromBytes => TryFromBytes for ManuallyDrop[T]); +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for ManuallyDrop[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for ManuallyDrop[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for ManuallyDrop[T]); +// SAFETY: `ManuallyDrop` has the same layout as `T` [1], and thus has the +// same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop` is guaranteed to have the same layout and bit validity as +// `T` +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for ManuallyDrop) }; +assert_unaligned!(ManuallyDrop<()>, ManuallyDrop); + +const _: () = { + #[allow( + non_camel_case_types, + missing_copy_implementations, + missing_debug_implementations, + missing_docs + )] + pub enum value {} + + // SAFETY: See safety comment on `ProjectToTag`. + unsafe impl HasTag for ManuallyDrop { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Tag = (); + + // SAFETY: It is trivially sound to project any pointer to a pointer to + // a type of size zero and alignment 1 (which `()` is [1]). Such a + // pointer will trivially satisfy its aliasing and validity requirements + // (since it has a zero-sized referent), and its alignment requirement + // (since it is aligned to 1). + // + // [1] Per https://doc.rust-lang.org/1.92.0/reference/type-layout.html#r-layout.tuple.unit: + // + // [T]he unit tuple (`()`)... is guaranteed as a zero-sized type to + // have a size of 0 and an alignment of 1. + type ProjectToTag = crate::pointer::cast::CastToUnit; + } + + // SAFETY: `ManuallyDrop` has a field of type `T` at offset `0` without + // any safety invariants beyond those of `T`. Its existence is not + // explicitly documented, but it can be inferred; per [1] `ManuallyDrop` + // has the same size and bit validity as `T`. This field is not literally + // public, but is effectively so; the field can be transparently: + // + // - initialized via `ManuallyDrop::new` + // - moved via `ManuallyDrop::into_inner` + // - referenced via `ManuallyDrop::deref` + // - exclusively referenced via `ManuallyDrop::deref_mut` + // + // We call this field `value`, both because that is both the name of this + // private field, and because it is the name it is referred to in the public + // documentation of `ManuallyDrop::new`, `ManuallyDrop::into_inner`, + // `ManuallyDrop::take` and `ManuallyDrop::drop`. + unsafe impl + HasField + for ManuallyDrop + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Type = T; + + #[inline(always)] + fn project(slf: PtrInner<'_, Self>) -> *mut T { + // SAFETY: `ManuallyDrop` has the same layout and bit validity as + // `T` [1]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop` is guaranteed to have the same layout and bit + // validity as `T` + #[allow(clippy::as_conversions)] + return slf.as_ptr() as *mut T; + } + } +}; + +impl_for_transmute_from!(T: ?Sized + TryFromBytes => TryFromBytes for Cell[T]); +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for Cell[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for Cell[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for Cell[T]); +// SAFETY: `Cell` has the same in-memory representation as `T` [1], and thus +// has the same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.Cell.html#memory-layout: +// +// `Cell` has the same in-memory representation as its inner type `T`. +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for Cell) }; + +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for UnsafeCell[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for UnsafeCell[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for UnsafeCell[T]); +// SAFETY: `UnsafeCell` has the same in-memory representation as `T` [1], and +// thus has the same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell` has the same in-memory representation as its inner type +// `T`. +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for UnsafeCell) }; +assert_unaligned!(UnsafeCell<()>, UnsafeCell); + +// SAFETY: See safety comment in `is_bit_valid` impl. +unsafe impl TryFromBytes for UnsafeCell { + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + #[inline(always)] + fn is_bit_valid(candidate: Maybe<'_, Self, A>) -> bool + where + A: invariant::Alignment, + { + T::is_bit_valid(candidate.transmute::<_, _, BecauseImmutable>()) + } +} + +// SAFETY: Per the reference [1]: +// +// An array of `[T; N]` has a size of `size_of::() * N` and the same +// alignment of `T`. Arrays are laid out so that the zero-based `nth` element +// of the array is offset from the start of the array by `n * size_of::()` +// bytes. +// +// ... +// +// Slices have the same layout as the section of the array they slice. +// +// In other words, the layout of a `[T]` or `[T; N]` is a sequence of `T`s laid +// out back-to-back with no bytes in between. Therefore, `[T]` or `[T; N]` are +// `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, and `IntoBytes` if `T` +// is (respectively). Furthermore, since an array/slice has "the same alignment +// of `T`", `[T]` and `[T; N]` are `Unaligned` if `T` is. +// +// Note that we don't `assert_unaligned!` for slice types because +// `assert_unaligned!` uses `align_of`, which only works for `Sized` types. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(const N: usize, T: Immutable => Immutable for [T; N]); + unsafe_impl!(const N: usize, T: TryFromBytes => TryFromBytes for [T; N]; |c| { + let c: Ptr<'_, [ReadOnly; N], _> = c.cast::<_, crate::pointer::cast::CastSized, _>(); + let c: Ptr<'_, [ReadOnly], _> = c.as_slice(); + let c: Ptr<'_, ReadOnly<[T]>, _> = c.cast::<_, crate::pointer::cast::CastUnsized, _>(); + + // Note that this call may panic, but it would still be sound even if it + // did. `is_bit_valid` does not promise that it will not panic (in fact, + // it explicitly warns that it's a possibility), and we have not + // violated any safety invariants that we must fix before returning. + <[T] as TryFromBytes>::is_bit_valid(c) + }); + unsafe_impl!(const N: usize, T: FromZeros => FromZeros for [T; N]); + unsafe_impl!(const N: usize, T: FromBytes => FromBytes for [T; N]); + unsafe_impl!(const N: usize, T: IntoBytes => IntoBytes for [T; N]); + unsafe_impl!(const N: usize, T: Unaligned => Unaligned for [T; N]); + assert_unaligned!([(); 0], [(); 1], [u8; 0], [u8; 1]); + unsafe_impl!(T: Immutable => Immutable for [T]); + unsafe_impl!(T: TryFromBytes => TryFromBytes for [T]; |c| { + let c: Ptr<'_, [ReadOnly], _> = c.cast::<_, crate::pointer::cast::CastUnsized, _>(); + + // SAFETY: Per the reference [1]: + // + // An array of `[T; N]` has a size of `size_of::() * N` and the + // same alignment of `T`. Arrays are laid out so that the zero-based + // `nth` element of the array is offset from the start of the array by + // `n * size_of::()` bytes. + // + // ... + // + // Slices have the same layout as the section of the array they slice. + // + // In other words, the layout of a `[T] is a sequence of `T`s laid out + // back-to-back with no bytes in between. If all elements in `candidate` + // are `is_bit_valid`, so too is `candidate`. + // + // Note that any of the below calls may panic, but it would still be + // sound even if it did. `is_bit_valid` does not promise that it will + // not panic (in fact, it explicitly warns that it's a possibility), and + // we have not violated any safety invariants that we must fix before + // returning. + c.iter().all(::is_bit_valid) + }); + unsafe_impl!(T: FromZeros => FromZeros for [T]); + unsafe_impl!(T: FromBytes => FromBytes for [T]); + unsafe_impl!(T: IntoBytes => IntoBytes for [T]); + unsafe_impl!(T: Unaligned => Unaligned for [T]); +}; + +// SAFETY: +// - `Immutable`: Raw pointers do not contain any `UnsafeCell`s. +// - `FromZeros`: For thin pointers (note that `T: Sized`), the zero pointer is +// considered "null". [1] No operations which require provenance are legal on +// null pointers, so this is not a footgun. +// - `TryFromBytes`: By the same reasoning as for `FromZeroes`, we can implement +// `TryFromBytes` for thin pointers provided that +// [`TryFromByte::is_bit_valid`] only produces `true` for zeroed bytes. +// +// NOTE(#170): Implementing `FromBytes` and `IntoBytes` for raw pointers would +// be sound, but carries provenance footguns. We want to support `FromBytes` and +// `IntoBytes` for raw pointers eventually, but we are holding off until we can +// figure out how to address those footguns. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/ptr/fn.null.html: +// +// Creates a null raw pointer. +// +// This function is equivalent to zero-initializing the pointer: +// `MaybeUninit::<*const T>::zeroed().assume_init()`. +// +// The resulting pointer has the address 0. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for *const T); + unsafe_impl!(T: ?Sized => Immutable for *mut T); + unsafe_impl!(T => TryFromBytes for *const T; |c| pointer::is_zeroed(c)); + unsafe_impl!(T => FromZeros for *const T); + unsafe_impl!(T => TryFromBytes for *mut T; |c| pointer::is_zeroed(c)); + unsafe_impl!(T => FromZeros for *mut T); +}; + +// SAFETY: `NonNull` self-evidently does not contain `UnsafeCell`s. This is +// not a proof, but we are accepting this as a known risk per #1358. +const _: () = unsafe { unsafe_impl!(T: ?Sized => Immutable for NonNull) }; + +// SAFETY: Reference types do not contain any `UnsafeCell`s. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for &'_ T); + unsafe_impl!(T: ?Sized => Immutable for &'_ mut T); +}; + +// SAFETY: `Option` is not `#[non_exhaustive]` [1], which means that the types +// in its variants cannot change, and no new variants can be added. `Option` +// does not contain any `UnsafeCell`s outside of `T`. [1] +// +// [1] https://doc.rust-lang.org/core/option/enum.Option.html +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for Option) }; + +mod tuples { + use super::*; + + /// Generates various trait implementations for tuples. + /// + /// # Safety + /// + /// `impl_tuple!` should be provided name-number pairs, where each number is + /// the ordinal of the preceding type name. + macro_rules! impl_tuple { + // Entry point. + ($($T:ident $I:tt),+ $(,)?) => { + crate::util::macros::__unsafe(); + impl_tuple!(@all [] [$($T $I)+]); + }; + + // Build up the set of tuple types (i.e., `(A,)`, `(A, B)`, `(A, B, C)`, + // etc.) Trait implementations that do not depend on field index may be + // added to this branch. + (@all [$($head_T:ident $head_I:tt)*] [$next_T:ident $next_I:tt $($tail:tt)*]) => { + // SAFETY: If all fields of the tuple `Self` are `Immutable`, so too is `Self`. + unsafe_impl!($($head_T: Immutable,)* $next_T: Immutable => Immutable for ($($head_T,)* $next_T,)); + + // SAFETY: If all fields in `c` are `is_bit_valid`, so too is `c`. + unsafe_impl!($($head_T: TryFromBytes,)* $next_T: TryFromBytes => TryFromBytes for ($($head_T,)* $next_T,); |c| { + let mut c = c; + $(TryFromBytes::is_bit_valid(into_inner!(c.reborrow().project::<_, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($head_I) }>())) &&)* + TryFromBytes::is_bit_valid(into_inner!(c.reborrow().project::<_, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($next_I) }>())) + }); + + // SAFETY: If all fields in `Self` are `FromZeros`, so too is `Self`. + unsafe_impl!($($head_T: FromZeros,)* $next_T: FromZeros => FromZeros for ($($head_T,)* $next_T,)); + + // SAFETY: If all fields in `Self` are `FromBytes`, so too is `Self`. + unsafe_impl!($($head_T: FromBytes,)* $next_T: FromBytes => FromBytes for ($($head_T,)* $next_T,)); + + // SAFETY: See safety comment on `ProjectToTag`. + unsafe impl<$($head_T,)* $next_T> crate::HasTag for ($($head_T,)* $next_T,) { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + type Tag = (); + + // SAFETY: It is trivially sound to project any pointer to a + // pointer to a type of size zero and alignment 1 (which `()` is + // [1]). Such a pointer will trivially satisfy its aliasing and + // validity requirements (since it has a zero-sized referent), + // and its alignment requirement (since it is aligned to 1). + // + // [1] Per https://doc.rust-lang.org/1.92.0/reference/type-layout.html#r-layout.tuple.unit: + // + // [T]he unit tuple (`()`)... is guaranteed as a zero-sized + // type to have a size of 0 and an alignment of 1. + type ProjectToTag = crate::pointer::cast::CastToUnit; + } + + // Generate impls that depend on tuple index. + impl_tuple!(@variants + [$($head_T $head_I)* $next_T $next_I] + [] + [$($head_T $head_I)* $next_T $next_I] + ); + + // Recurse to next tuple size + impl_tuple!(@all [$($head_T $head_I)* $next_T $next_I] [$($tail)*]); + }; + (@all [$($head_T:ident $head_I:tt)*] []) => {}; + + // Emit trait implementations that depend on field index. + (@variants + // The full tuple definition in type–index pairs. + [$($AllT:ident $AllI:tt)+] + // Types before the current index. + [$($BeforeT:ident)*] + // The types and indices at and after the current index. + [$CurrT:ident $CurrI:tt $($AfterT:ident $AfterI:tt)*] + ) => { + // SAFETY: + // - `Self` is a struct (albeit anonymous), so `VARIANT_ID` is + // `STRUCT_VARIANT_ID`. + // - `$CurrI` is the field at index `$CurrI`, so `FIELD_ID` is + // `zerocopy::ident_id!($CurrI)` + // - `()` has the same visibility as the `.$CurrI` field (ie, `.0`, + // `.1`, etc) + // - `Type` has the same type as `$CurrI`; i.e., `$CurrT`. + unsafe impl<$($AllT),+> crate::HasField< + (), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + type Type = $CurrT; + + #[inline(always)] + fn project(slf: crate::PtrInner<'_, Self>) -> *mut Self::Type { + let slf = slf.as_non_null().as_ptr(); + // SAFETY: `PtrInner` promises it references either a zero-sized + // byte range, or else will reference a byte range that is + // entirely contained within an allocated object. In either + // case, this guarantees that `(*slf).$CurrI` is in-bounds of + // `slf`. + unsafe { core::ptr::addr_of_mut!((*slf).$CurrI) } + } + } + + // SAFETY: See comments on items. + unsafe impl crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Uninit), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Uninit); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // SAFETY: See comments on items. + unsafe impl crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Initialized), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Initialized); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // SAFETY: See comments on items. + unsafe impl crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Valid), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Valid); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // Recurse to the next index. + impl_tuple!(@variants [$($AllT $AllI)+] [$($BeforeT)* $CurrT] [$($AfterT $AfterI)*]); + }; + (@variants [$($AllT:ident $AllI:tt)+] [$($BeforeT:ident)*] []) => {}; + } + + // SAFETY: `impl_tuple` is provided name-number pairs, where number is the + // ordinal of the name. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + impl_tuple! { + A 0, + B 1, + C 2, + D 3, + E 4, + F 5, + G 6, + H 7, + I 8, + J 9, + K 10, + L 11, + M 12, + N 13, + O 14, + P 15, + Q 16, + R 17, + S 18, + T 19, + U 20, + V 21, + W 22, + X 23, + Y 24, + Z 25, + }; + }; +} + +// SIMD support +// +// Per the Unsafe Code Guidelines Reference [1]: +// +// Packed SIMD vector types are `repr(simd)` homogeneous tuple-structs +// containing `N` elements of type `T` where `N` is a power-of-two and the +// size and alignment requirements of `T` are equal: +// +// ```rust +// #[repr(simd)] +// struct Vector(T_0, ..., T_(N - 1)); +// ``` +// +// ... +// +// The size of `Vector` is `N * size_of::()` and its alignment is an +// implementation-defined function of `T` and `N` greater than or equal to +// `align_of::()`. +// +// ... +// +// Vector elements are laid out in source field order, enabling random access +// to vector elements by reinterpreting the vector as an array: +// +// ```rust +// union U { +// vec: Vector, +// arr: [T; N] +// } +// +// assert_eq!(size_of::>(), size_of::<[T; N]>()); +// assert!(align_of::>() >= align_of::<[T; N]>()); +// +// unsafe { +// let u = U { vec: Vector(t_0, ..., t_(N - 1)) }; +// +// assert_eq!(u.vec.0, u.arr[0]); +// // ... +// assert_eq!(u.vec.(N - 1), u.arr[N - 1]); +// } +// ``` +// +// Given this background, we can observe that: +// - The size and bit pattern requirements of a SIMD type are equivalent to the +// equivalent array type. Thus, for any SIMD type whose primitive `T` is +// `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, or `IntoBytes`, that +// SIMD type is also `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, or +// `IntoBytes` respectively. +// - Since no upper bound is placed on the alignment, no SIMD type can be +// guaranteed to be `Unaligned`. +// +// Also per [1]: +// +// This chapter represents the consensus from issue #38. The statements in +// here are not (yet) "guaranteed" not to change until an RFC ratifies them. +// +// See issue #38 [2]. While this behavior is not technically guaranteed, the +// likelihood that the behavior will change such that SIMD types are no longer +// `TryFromBytes`, `FromZeros`, `FromBytes`, or `IntoBytes` is next to zero, as +// that would defeat the entire purpose of SIMD types. Nonetheless, we put this +// behavior behind the `simd` Cargo feature, which requires consumers to opt +// into this stability hazard. +// +// [1] https://rust-lang.github.io/unsafe-code-guidelines/layout/packed-simd-vectors.html +// [2] https://github.com/rust-lang/unsafe-code-guidelines/issues/38 +#[cfg(feature = "simd")] +#[cfg_attr(doc_cfg, doc(cfg(feature = "simd")))] +mod simd { + /// Defines a module which implements `TryFromBytes`, `FromZeros`, + /// `FromBytes`, and `IntoBytes` for a set of types from a module in + /// `core::arch`. + /// + /// `$arch` is both the name of the defined module and the name of the + /// module in `core::arch`, and `$typ` is the list of items from that module + /// to implement `FromZeros`, `FromBytes`, and `IntoBytes` for. + #[allow(unused_macros)] // `allow(unused_macros)` is needed because some + // target/feature combinations don't emit any impls + // and thus don't use this macro. + macro_rules! simd_arch_mod { + ($(#[cfg $cfg:tt])* $(#[cfg_attr $cfg_attr:tt])? $arch:ident, $mod:ident, $($typ:ident),*) => { + $(#[cfg $cfg])* + #[cfg_attr(doc_cfg, doc(cfg $($cfg)*))] + $(#[cfg_attr $cfg_attr])? + mod $mod { + use core::arch::$arch::{$($typ),*}; + + use crate::*; + impl_known_layout!($($typ),*); + // SAFETY: See comment on module definition for justification. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + $( unsafe_impl!($typ: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); )* + }; + } + }; + } + + #[rustfmt::skip] + const _: () = { + simd_arch_mod!( + #[cfg(target_arch = "x86")] + x86, x86, __m128, __m128d, __m128i, __m256, __m256d, __m256i + ); + #[cfg(not(no_zerocopy_simd_x86_avx12_1_89_0))] + simd_arch_mod!( + #[cfg(target_arch = "x86")] + #[cfg_attr(doc_cfg, doc(cfg(rust = "1.89.0")))] + x86, x86_nightly, __m512bh, __m512, __m512d, __m512i + ); + simd_arch_mod!( + #[cfg(target_arch = "x86_64")] + x86_64, x86_64, __m128, __m128d, __m128i, __m256, __m256d, __m256i + ); + #[cfg(not(no_zerocopy_simd_x86_avx12_1_89_0))] + simd_arch_mod!( + #[cfg(target_arch = "x86_64")] + #[cfg_attr(doc_cfg, doc(cfg(rust = "1.89.0")))] + x86_64, x86_64_nightly, __m512bh, __m512, __m512d, __m512i + ); + simd_arch_mod!( + #[cfg(target_arch = "wasm32")] + wasm32, wasm32, v128 + ); + simd_arch_mod!( + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc"))] + powerpc, powerpc, vector_bool_long, vector_double, vector_signed_long, vector_unsigned_long + ); + simd_arch_mod!( + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc64"))] + powerpc64, powerpc64, vector_bool_long, vector_double, vector_signed_long, vector_unsigned_long + ); + // NOTE: NEON intrinsics were broken on big-endian platforms from their stabilization up to + // Rust 1.87. (Context in https://github.com/rust-lang/stdarch/issues/1484). Support is + // split in two different version ranges on top of the base configuration, requiring either + // little endian or the more recent version to be detected as well. + #[cfg(not(no_zerocopy_aarch64_simd_1_59_0))] + simd_arch_mod!( + #[cfg(all( + target_arch = "aarch64", + any( + target_endian = "little", + not(no_zerocopy_aarch64_simd_be_1_87_0) + ) + ))] + #[cfg_attr( + doc_cfg, + doc(cfg(all(target_arch = "aarch64", any( + all(rust = "1.59.0", target_endian = "little"), + rust = "1.87.0", + )))) + )] + aarch64, aarch64, float32x2_t, float32x4_t, float64x1_t, float64x2_t, int8x8_t, int8x8x2_t, + int8x8x3_t, int8x8x4_t, int8x16_t, int8x16x2_t, int8x16x3_t, int8x16x4_t, int16x4_t, + int16x8_t, int32x2_t, int32x4_t, int64x1_t, int64x2_t, poly8x8_t, poly8x8x2_t, poly8x8x3_t, + poly8x8x4_t, poly8x16_t, poly8x16x2_t, poly8x16x3_t, poly8x16x4_t, poly16x4_t, poly16x8_t, + poly64x1_t, poly64x2_t, uint8x8_t, uint8x8x2_t, uint8x8x3_t, uint8x8x4_t, uint8x16_t, + uint8x16x2_t, uint8x16x3_t, uint8x16x4_t, uint16x4_t, uint16x4x2_t, uint16x4x3_t, + uint16x4x4_t, uint16x8_t, uint32x2_t, uint32x4_t, uint64x1_t, uint64x2_t + ); + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_impls() { + // A type that can supply test cases for testing + // `TryFromBytes::is_bit_valid`. All types passed to `assert_impls!` + // must implement this trait; that macro uses it to generate runtime + // tests for `TryFromBytes` impls. + // + // All `T: FromBytes` types are provided with a blanket impl. Other + // types must implement `TryFromBytesTestable` directly (ie using + // `impl_try_from_bytes_testable!`). + trait TryFromBytesTestable { + fn with_passing_test_cases>)>(f: F); + fn with_failing_test_cases(f: F); + } + + impl TryFromBytesTestable for T { + fn with_passing_test_cases>)>(f: F) { + // Test with a zeroed value. + f(ReadOnly::::new_box_zeroed().unwrap()); + + let ffs = { + let mut t = ReadOnly::new(Self::new_zeroed()); + let ptr: *mut T = ReadOnly::as_mut(&mut t); + // SAFETY: `T: FromBytes` + unsafe { ptr::write_bytes(ptr.cast::(), 0xFF, mem::size_of::()) }; + t + }; + + // Test with a value initialized with 0xFF. + f(Box::new(ffs)); + } + + fn with_failing_test_cases(_f: F) {} + } + + macro_rules! impl_try_from_bytes_testable_for_null_pointer_optimization { + ($($tys:ty),*) => { + $( + impl TryFromBytesTestable for Option<$tys> { + fn with_passing_test_cases>)>(f: F) { + // Test with a zeroed value. + f(Box::new(ReadOnly::new(None))); + } + + fn with_failing_test_cases(f: F) { + for pos in 0..mem::size_of::() { + let mut bytes = [0u8; mem::size_of::()]; + bytes[pos] = 0x01; + f(&mut bytes[..]); + } + } + } + )* + }; + } + + // Implements `TryFromBytesTestable`. + macro_rules! impl_try_from_bytes_testable { + // Base case for recursion (when the list of types has run out). + (=> @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => {}; + // Implements for type(s) with no type parameters. + ($ty:ty $(,$tys:ty)* => @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => { + impl TryFromBytesTestable for $ty { + impl_try_from_bytes_testable!( + @methods @success $($success_case),* + $(, @failure $($failure_case),*)? + ); + } + impl_try_from_bytes_testable!($($tys),* => @success $($success_case),* $(, @failure $($failure_case),*)?); + }; + // Implements for multiple types with no type parameters. + ($($($ty:ty),* => @success $($success_case:expr), * $(, @failure $($failure_case:expr),*)?;)*) => { + $( + impl_try_from_bytes_testable!($($ty),* => @success $($success_case),* $(, @failure $($failure_case),*)*); + )* + }; + // Implements only the methods; caller must invoke this from inside + // an impl block. + (@methods @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => { + fn with_passing_test_cases>)>(_f: F) { + $( + let bx = Box::::from($success_case); + let ro: Box> = { + let raw = Box::into_raw(bx); + // SAFETY: `ReadOnly` has the same layout and bit + // validity as `T`. + #[allow(clippy::as_conversions)] + unsafe { Box::from_raw(raw as *mut _) } + }; + _f(ro); + )* + } + + fn with_failing_test_cases(_f: F) { + $($( + let mut case = $failure_case; + _f(case.as_mut_bytes()); + )*)? + } + }; + } + + impl_try_from_bytes_testable_for_null_pointer_optimization!( + Box>, + &'static UnsafeCell, + &'static mut UnsafeCell, + NonNull>, + fn(), + FnManyArgs, + extern "C" fn(), + ECFnManyArgs + ); + + macro_rules! bx { + ($e:expr) => { + Box::new($e) + }; + } + + // Note that these impls are only for types which are not `FromBytes`. + // `FromBytes` types are covered by a preceding blanket impl. + impl_try_from_bytes_testable!( + bool => @success true, false, + @failure 2u8, 3u8, 0xFFu8; + char => @success '\u{0}', '\u{D7FF}', '\u{E000}', '\u{10FFFF}', + @failure 0xD800u32, 0xDFFFu32, 0x110000u32; + str => @success "", "hello", "❤️🧡💛💚💙💜", + @failure [0, 159, 146, 150]; + [u8] => @success vec![].into_boxed_slice(), vec![0, 1, 2].into_boxed_slice(); + NonZeroU8, NonZeroI8, NonZeroU16, NonZeroI16, NonZeroU32, + NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, + NonZeroUsize, NonZeroIsize + => @success Self::new(1).unwrap(), + // Doing this instead of `0` ensures that we always satisfy + // the size and alignment requirements of `Self` (whereas `0` + // may be any integer type with a different size or alignment + // than some `NonZeroXxx` types). + @failure Option::::None; + [bool; 0] => @success []; + [bool; 1] + => @success [true], [false], + @failure [2u8], [3u8], [0xFFu8]; + [bool] + => @success vec![true, false].into_boxed_slice(), vec![false, true].into_boxed_slice(), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + Unalign + => @success Unalign::new(false), Unalign::new(true), + @failure 2u8, 0xFFu8; + ManuallyDrop + => @success ManuallyDrop::new(false), ManuallyDrop::new(true), + @failure 2u8, 0xFFu8; + ManuallyDrop<[u8]> + => @success bx!(ManuallyDrop::new([])), bx!(ManuallyDrop::new([0u8])), bx!(ManuallyDrop::new([0u8, 1u8])); + ManuallyDrop<[bool]> + => @success bx!(ManuallyDrop::new([])), bx!(ManuallyDrop::new([false])), bx!(ManuallyDrop::new([false, true])), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + ManuallyDrop<[UnsafeCell]> + => @success bx!(ManuallyDrop::new([UnsafeCell::new(0)])), bx!(ManuallyDrop::new([UnsafeCell::new(0), UnsafeCell::new(1)])); + ManuallyDrop<[UnsafeCell]> + => @success bx!(ManuallyDrop::new([UnsafeCell::new(false)])), bx!(ManuallyDrop::new([UnsafeCell::new(false), UnsafeCell::new(true)])), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + Wrapping + => @success Wrapping(false), Wrapping(true), + @failure 2u8, 0xFFu8; + *const NotZerocopy + => @success ptr::null::(), + @failure [0x01; mem::size_of::<*const NotZerocopy>()]; + *mut NotZerocopy + => @success ptr::null_mut::(), + @failure [0x01; mem::size_of::<*mut NotZerocopy>()]; + ); + + // Use the trick described in [1] to allow us to call methods + // conditional on certain trait bounds. + // + // In all of these cases, methods return `Option`, where `R` is the + // return type of the method we're conditionally calling. The "real" + // implementations (the ones defined in traits using `&self`) return + // `Some`, and the default implementations (the ones defined as inherent + // methods using `&mut self`) return `None`. + // + // [1] https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md + mod autoref_trick { + use super::*; + + pub(super) struct AutorefWrapper(pub(super) PhantomData); + + pub(super) trait TestIsBitValidShared { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>(&self, candidate: Maybe<'ptr, T>) + -> Option; + } + + impl TestIsBitValidShared for AutorefWrapper { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>( + &self, + candidate: Maybe<'ptr, T>, + ) -> Option { + Some(T::is_bit_valid(candidate)) + } + } + + pub(super) trait TestTryFromRef { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>( + &self, + bytes: &'bytes [u8], + ) -> Option>; + } + + impl TestTryFromRef for AutorefWrapper { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>( + &self, + bytes: &'bytes [u8], + ) -> Option> { + Some(T::try_ref_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestTryFromMut { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>( + &self, + bytes: &'bytes mut [u8], + ) -> Option>; + } + + impl TestTryFromMut for AutorefWrapper { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>( + &self, + bytes: &'bytes mut [u8], + ) -> Option> { + Some(T::try_mut_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestTryReadFrom { + fn test_try_read_from(&self, bytes: &[u8]) -> Option>; + } + + impl TestTryReadFrom for AutorefWrapper { + fn test_try_read_from(&self, bytes: &[u8]) -> Option> { + Some(T::try_read_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestAsBytes { + #[allow(clippy::needless_lifetimes)] + fn test_as_bytes<'slf, 't>(&'slf self, t: &'t ReadOnly) -> Option<&'t [u8]>; + } + + impl TestAsBytes for AutorefWrapper { + #[allow(clippy::needless_lifetimes)] + fn test_as_bytes<'slf, 't>(&'slf self, t: &'t ReadOnly) -> Option<&'t [u8]> { + Some(t.as_bytes()) + } + } + } + + use autoref_trick::*; + + // Asserts that `$ty` is one of a list of types which are allowed to not + // provide a "real" implementation for `$fn_name`. Since the + // `autoref_trick` machinery fails silently, this allows us to ensure + // that the "default" impls are only being used for types which we + // expect. + // + // Note that, since this is a runtime test, it is possible to have an + // allowlist which is too restrictive if the function in question is + // never called for a particular type. For example, if `as_bytes` is not + // supported for a particular type, and so `test_as_bytes` returns + // `None`, methods such as `test_try_from_ref` may never be called for + // that type. As a result, it's possible that, for example, adding + // `as_bytes` support for a type would cause other allowlist assertions + // to fail. This means that allowlist assertion failures should not + // automatically be taken as a sign of a bug. + macro_rules! assert_on_allowlist { + ($fn_name:ident($ty:ty) $(: $($tys:ty),*)?) => {{ + use core::any::TypeId; + + let allowlist: &[TypeId] = &[ $($(TypeId::of::<$tys>()),*)? ]; + let allowlist_names: &[&str] = &[ $($(stringify!($tys)),*)? ]; + + let id = TypeId::of::<$ty>(); + assert!(allowlist.contains(&id), "{} is not on allowlist for {}: {:?}", stringify!($ty), stringify!($fn_name), allowlist_names); + }}; + } + + // Asserts that `$ty` implements any `$trait` and doesn't implement any + // `!$trait`. Note that all `$trait`s must come before any `!$trait`s. + // + // For `T: TryFromBytes`, uses `TryFromBytesTestable` to test success + // and failure cases. + macro_rules! assert_impls { + ($ty:ty: TryFromBytes) => { + // "Default" implementations that match the "real" + // implementations defined in the `autoref_trick` module above. + #[allow(unused, non_local_definitions)] + impl AutorefWrapper<$ty> { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>( + &mut self, + candidate: Maybe<'ptr, $ty>, + ) -> Option { + assert_on_allowlist!( + test_is_bit_valid_shared($ty): + ManuallyDrop>, + ManuallyDrop<[UnsafeCell]>, + ManuallyDrop<[UnsafeCell]>, + CoreMaybeUninit, + CoreMaybeUninit>, + Wrapping> + ); + + None + } + + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>(&mut self, _bytes: &'bytes [u8]) -> Option> { + assert_on_allowlist!( + test_try_from_ref($ty): + ManuallyDrop<[UnsafeCell]> + ); + + None + } + + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>(&mut self, _bytes: &'bytes mut [u8]) -> Option> { + assert_on_allowlist!( + test_try_from_mut($ty): + Option>>, + Option<&'static UnsafeCell>, + Option<&'static mut UnsafeCell>, + Option>>, + Option, + Option, + Option, + Option, + *const NotZerocopy, + *mut NotZerocopy + ); + + None + } + + fn test_try_read_from(&mut self, _bytes: &[u8]) -> Option> { + assert_on_allowlist!( + test_try_read_from($ty): + str, + ManuallyDrop<[u8]>, + ManuallyDrop<[bool]>, + ManuallyDrop<[UnsafeCell]>, + [u8], + [bool] + ); + + None + } + + fn test_as_bytes(&mut self, _t: &ReadOnly<$ty>) -> Option<&[u8]> { + assert_on_allowlist!( + test_as_bytes($ty): + Option<&'static UnsafeCell>, + Option<&'static mut UnsafeCell>, + Option>>, + Option>>, + Option, + Option, + Option, + Option, + CoreMaybeUninit, + CoreMaybeUninit, + CoreMaybeUninit>, + ManuallyDrop>, + ManuallyDrop<[UnsafeCell]>, + ManuallyDrop<[UnsafeCell]>, + Wrapping>, + *const NotZerocopy, + *mut NotZerocopy + ); + + None + } + } + + <$ty as TryFromBytesTestable>::with_passing_test_cases(|mut val| { + // FIXME(#494): These tests only get exercised for types + // which are `IntoBytes`. Once we implement #494, we should + // be able to support non-`IntoBytes` types by zeroing + // padding. + + // We define `w` and `ww` since, in the case of the inherent + // methods, Rust thinks they're both borrowed mutably at the + // same time (given how we use them below). If we just + // defined a single `w` and used it for multiple operations, + // this would conflict. + // + // We `#[allow(unused_mut]` for the cases where the "real" + // impls are used, which take `&self`. + #[allow(unused_mut)] + let (mut w, mut ww) = (AutorefWrapper::<$ty>(PhantomData), AutorefWrapper::<$ty>(PhantomData)); + + let c = Ptr::from_ref(&*val); + let c = c.forget_aligned(); + // SAFETY: FIXME(#899): This is unsound. `$ty` is not + // necessarily `IntoBytes`, but that's the corner we've + // backed ourselves into by using `Ptr::from_ref`. + let c = unsafe { c.assume_initialized() }; + let res = w.test_is_bit_valid_shared(c); + if let Some(res) = res { + assert!(res, "{}::is_bit_valid (shared `Ptr`): got false, expected true", stringify!($ty)); + } + + let c = Ptr::from_mut(&mut *val); + let c = c.forget_aligned(); + // SAFETY: FIXME(#899): This is unsound. `$ty` is not + // necessarily `IntoBytes`, but that's the corner we've + // backed ourselves into by using `Ptr::from_ref`. + let mut c = unsafe { c.assume_initialized() }; + let res = <$ty as TryFromBytes>::is_bit_valid(c.reborrow_shared()); + assert!(res, "{}::is_bit_valid (exclusive `Ptr`): got false, expected true", stringify!($ty)); + + // `bytes` is `Some(val.as_bytes())` if `$ty: IntoBytes + + // Immutable` and `None` otherwise. + let bytes = w.test_as_bytes(&*val); + + // The inner closure returns + // `Some($ty::try_ref_from_bytes(bytes))` if `$ty: + // Immutable` and `None` otherwise. + let res = bytes.and_then(|bytes| ww.test_try_from_ref(bytes)); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_ref_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + + if let Some(bytes) = bytes { + // We need to get a mutable byte slice, and so we clone + // into a `Vec`. However, we also need these bytes to + // satisfy `$ty`'s alignment requirement, which isn't + // guaranteed for `Vec`. In order to get around + // this, we create a `Vec` which is twice as long as we + // need. There is guaranteed to be an aligned byte range + // of size `size_of_val(val)` within that range. + let val = &*val; + let size = mem::size_of_val(val); + let align = mem::align_of_val(val); + + let mut vec = bytes.to_vec(); + vec.extend(bytes); + let slc = vec.as_slice(); + let offset = slc.as_ptr().align_offset(align); + let bytes_mut = &mut vec.as_mut_slice()[offset..offset+size]; + bytes_mut.copy_from_slice(bytes); + + let res = ww.test_try_from_mut(bytes_mut); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_mut_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + } + + let res = bytes.and_then(|bytes| ww.test_try_read_from(bytes)); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_read_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + }); + #[allow(clippy::as_conversions)] + <$ty as TryFromBytesTestable>::with_failing_test_cases(|c| { + #[allow(unused_mut)] // For cases where the "real" impls are used, which take `&self`. + let mut w = AutorefWrapper::<$ty>(PhantomData); + + // This is `Some($ty::try_ref_from_bytes(c))` if `$ty: + // Immutable` and `None` otherwise. + let res = w.test_try_from_ref(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_ref_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + + let res = w.test_try_from_mut(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_mut_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + + + let res = w.test_try_read_from(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_read_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + }); + + #[allow(dead_code)] + const _: () = { static_assertions::assert_impl_all!($ty: TryFromBytes); }; + }; + ($ty:ty: $trait:ident) => { + #[allow(dead_code)] + const _: () = { static_assertions::assert_impl_all!($ty: $trait); }; + }; + ($ty:ty: !$trait:ident) => { + #[allow(dead_code)] + const _: () = { static_assertions::assert_not_impl_any!($ty: $trait); }; + }; + ($ty:ty: $($trait:ident),* $(,)? $(!$negative_trait:ident),*) => { + $( + assert_impls!($ty: $trait); + )* + + $( + assert_impls!($ty: !$negative_trait); + )* + }; + } + + // NOTE: The negative impl assertions here are not necessarily + // prescriptive. They merely serve as change detectors to make sure + // we're aware of what trait impls are getting added with a given + // change. Of course, some impls would be invalid (e.g., `bool: + // FromBytes`), and so this change detection is very important. + + assert_impls!( + (): KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + u8: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + i8: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + u16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + usize: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + isize: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + #[cfg(feature = "float-nightly")] + assert_impls!( + f16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + f32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + f64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + #[cfg(feature = "float-nightly")] + assert_impls!( + f128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + bool: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + assert_impls!( + char: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + str: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + + assert_impls!( + NonZeroU8: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + Unaligned, + !FromZeros, + !FromBytes + ); + assert_impls!( + NonZeroI8: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + Unaligned, + !FromZeros, + !FromBytes + ); + assert_impls!( + NonZeroU16: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI16: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU32: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI32: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU64: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI64: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU128: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI128: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroUsize: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroIsize: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + + // Implements none of the ZC traits. + struct NotZerocopy; + + #[rustfmt::skip] + type FnManyArgs = fn( + NotZerocopy, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, + ) -> (NotZerocopy, NotZerocopy); + + // Allowed, because we're not actually using this type for FFI. + #[allow(improper_ctypes_definitions)] + #[rustfmt::skip] + type ECFnManyArgs = extern "C" fn( + NotZerocopy, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, + ) -> (NotZerocopy, NotZerocopy); + + #[cfg(feature = "alloc")] + assert_impls!(Option>>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option]>>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static UnsafeCell>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static [UnsafeCell]>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static mut UnsafeCell>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static mut [UnsafeCell]>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option>>: KnownLayout, TryFromBytes, FromZeros, Immutable, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option]>>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + + assert_impls!(PhantomData: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(PhantomData>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(PhantomData<[u8]>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + + assert_impls!(ManuallyDrop: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of ` as TryFromBytes>::is_bit_valid`. + assert_impls!(ManuallyDrop: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(ManuallyDrop<[u8]>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of ` as TryFromBytes>::is_bit_valid`. + assert_impls!(ManuallyDrop<[bool]>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(ManuallyDrop: !Immutable, !TryFromBytes, !KnownLayout, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(ManuallyDrop<[NotZerocopy]>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(ManuallyDrop>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + assert_impls!(ManuallyDrop<[UnsafeCell]>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + assert_impls!(ManuallyDrop<[UnsafeCell]>: KnownLayout, TryFromBytes, FromZeros, IntoBytes, Unaligned, !Immutable, !FromBytes); + + assert_impls!(CoreMaybeUninit: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, Unaligned, !IntoBytes); + assert_impls!(CoreMaybeUninit: KnownLayout, TryFromBytes, FromZeros, FromBytes, !Immutable, !IntoBytes, !Unaligned); + assert_impls!(CoreMaybeUninit>: KnownLayout, TryFromBytes, FromZeros, FromBytes, Unaligned, !Immutable, !IntoBytes); + + assert_impls!(Wrapping: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of ` as TryFromBytes>::is_bit_valid`. + assert_impls!(Wrapping: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(Wrapping: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Wrapping>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + + assert_impls!(Unalign: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of ` as TryFromBytes>::is_bit_valid`. + assert_impls!(Unalign: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(Unalign: KnownLayout, Unaligned, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes); + + assert_impls!( + [u8]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + [bool]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + assert_impls!([NotZerocopy]: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!( + [u8; 0]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned, + ); + assert_impls!( + [NotZerocopy; 0]: KnownLayout, + !Immutable, + !TryFromBytes, + !FromZeros, + !FromBytes, + !IntoBytes, + !Unaligned + ); + assert_impls!( + [u8; 1]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned, + ); + assert_impls!( + [NotZerocopy; 1]: KnownLayout, + !Immutable, + !TryFromBytes, + !FromZeros, + !FromBytes, + !IntoBytes, + !Unaligned + ); + + assert_impls!(*const NotZerocopy: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut NotZerocopy: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*const [NotZerocopy]: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut [NotZerocopy]: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*const dyn Debug: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut dyn Debug: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + + #[cfg(feature = "simd")] + { + #[allow(unused_macros)] + macro_rules! test_simd_arch_mod { + ($arch:ident, $($typ:ident),*) => { + { + use core::arch::$arch::{$($typ),*}; + use crate::*; + $( assert_impls!($typ: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); )* + } + }; + } + #[cfg(target_arch = "x86")] + test_simd_arch_mod!(x86, __m128, __m128d, __m128i, __m256, __m256d, __m256i); + + #[cfg(all(not(no_zerocopy_simd_x86_avx12_1_89_0), target_arch = "x86"))] + test_simd_arch_mod!(x86, __m512bh, __m512, __m512d, __m512i); + + #[cfg(target_arch = "x86_64")] + test_simd_arch_mod!(x86_64, __m128, __m128d, __m128i, __m256, __m256d, __m256i); + + #[cfg(all(not(no_zerocopy_simd_x86_avx12_1_89_0), target_arch = "x86_64"))] + test_simd_arch_mod!(x86_64, __m512bh, __m512, __m512d, __m512i); + + #[cfg(target_arch = "wasm32")] + test_simd_arch_mod!(wasm32, v128); + + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc"))] + test_simd_arch_mod!( + powerpc, + vector_bool_long, + vector_double, + vector_signed_long, + vector_unsigned_long + ); + + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc64"))] + test_simd_arch_mod!( + powerpc64, + vector_bool_long, + vector_double, + vector_signed_long, + vector_unsigned_long + ); + #[cfg(all(target_arch = "aarch64", not(no_zerocopy_aarch64_simd_1_59_0)))] + #[rustfmt::skip] + test_simd_arch_mod!( + aarch64, float32x2_t, float32x4_t, float64x1_t, float64x2_t, int8x8_t, int8x8x2_t, + int8x8x3_t, int8x8x4_t, int8x16_t, int8x16x2_t, int8x16x3_t, int8x16x4_t, int16x4_t, + int16x8_t, int32x2_t, int32x4_t, int64x1_t, int64x2_t, poly8x8_t, poly8x8x2_t, poly8x8x3_t, + poly8x8x4_t, poly8x16_t, poly8x16x2_t, poly8x16x3_t, poly8x16x4_t, poly16x4_t, poly16x8_t, + poly64x1_t, poly64x2_t, uint8x8_t, uint8x8x2_t, uint8x8x3_t, uint8x8x4_t, uint8x16_t, + uint8x16x2_t, uint8x16x3_t, uint8x16x4_t, uint16x4_t, uint16x4x2_t, uint16x4x3_t, + uint16x4x4_t, uint16x8_t, uint32x2_t, uint32x4_t, uint64x1_t, uint64x2_t + ); + } + } +} diff --git a/rust/zerocopy/src/layout.rs b/rust/zerocopy/src/layout.rs new file mode 100644 index 000000000000..19ad5ca85f74 --- /dev/null +++ b/rust/zerocopy/src/layout.rs @@ -0,0 +1,2223 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{mem, num::NonZeroUsize}; + +use crate::util; + +/// The target pointer width, counted in bits. +const POINTER_WIDTH_BITS: usize = mem::size_of::() * 8; + +/// The layout of a type which might be dynamically-sized. +/// +/// `DstLayout` describes the layout of sized types, slice types, and "slice +/// DSTs" - ie, those that are known by the type system to have a trailing slice +/// (as distinguished from `dyn Trait` types - such types *might* have a +/// trailing slice type, but the type system isn't aware of it). +/// +/// Note that `DstLayout` does not have any internal invariants, so no guarantee +/// is made that a `DstLayout` conforms to any of Rust's requirements regarding +/// the layout of real Rust types or instances of types. +#[doc(hidden)] +#[allow(missing_debug_implementations, missing_copy_implementations)] +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub struct DstLayout { + pub(crate) align: NonZeroUsize, + pub(crate) size_info: SizeInfo, + // Is it guaranteed statically (without knowing a value's runtime metadata) + // that the top-level type contains no padding? This does *not* apply + // recursively - for example, `[(u8, u16)]` has `statically_shallow_unpadded + // = true` even though this type likely has padding inside each `(u8, u16)`. + pub(crate) statically_shallow_unpadded: bool, +} + +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub(crate) enum SizeInfo { + Sized { size: usize }, + SliceDst(TrailingSliceLayout), +} + +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub(crate) struct TrailingSliceLayout { + // The offset of the first byte of the trailing slice field. Note that this + // is NOT the same as the minimum size of the type. For example, consider + // the following type: + // + // struct Foo { + // a: u16, + // b: u8, + // c: [u8], + // } + // + // In `Foo`, `c` is at byte offset 3. When `c.len() == 0`, `c` is followed + // by a padding byte. + pub(crate) offset: usize, + // The size of the element type of the trailing slice field. + pub(crate) elem_size: E, +} + +impl SizeInfo { + /// Attempts to create a `SizeInfo` from `Self` in which `elem_size` is a + /// `NonZeroUsize`. If `elem_size` is 0, returns `None`. + #[allow(unused)] + const fn try_to_nonzero_elem_size(&self) -> Option> { + Some(match *self { + SizeInfo::Sized { size } => SizeInfo::Sized { size }, + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + if let Some(elem_size) = NonZeroUsize::new(elem_size) { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } else { + return None; + } + } + }) + } +} + +#[doc(hidden)] +#[derive(Copy, Clone)] +#[cfg_attr(test, derive(Debug))] +#[allow(missing_debug_implementations)] +pub enum CastType { + Prefix, + Suffix, +} + +#[cfg_attr(test, derive(Debug))] +pub(crate) enum MetadataCastError { + Alignment, + Size, +} + +impl DstLayout { + /// The minimum possible alignment of a type. + const MIN_ALIGN: NonZeroUsize = match NonZeroUsize::new(1) { + Some(min_align) => min_align, + None => const_unreachable!(), + }; + + /// The maximum theoretic possible alignment of a type. + /// + /// For compatibility with future Rust versions, this is defined as the + /// maximum power-of-two that fits into a `usize`. See also + /// [`DstLayout::CURRENT_MAX_ALIGN`]. + pub(crate) const THEORETICAL_MAX_ALIGN: NonZeroUsize = + match NonZeroUsize::new(1 << (POINTER_WIDTH_BITS - 1)) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + /// The current, documented max alignment of a type \[1\]. + /// + /// \[1\] Per : + /// + /// The alignment value must be a power of two from 1 up to + /// 229. + #[cfg(not(kani))] + #[cfg(not(target_pointer_width = "16"))] + pub(crate) const CURRENT_MAX_ALIGN: NonZeroUsize = match NonZeroUsize::new(1 << 28) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + #[cfg(not(kani))] + #[cfg(target_pointer_width = "16")] + pub(crate) const CURRENT_MAX_ALIGN: NonZeroUsize = match NonZeroUsize::new(1 << 15) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + /// The maximum size of an allocation \[1\]. + /// + /// \[1\] Per : + /// + /// For any allocation with base `address`, `size`, and a set of `addresses`, + /// the following are guaranteed: [..] + /// + /// - `size <= isize::MAX` + /// + #[allow(clippy::as_conversions)] + pub(crate) const MAX_SIZE: usize = isize::MAX as usize; + + /// Assumes that this layout lacks static shallow padding. + /// + /// # Panics + /// + /// This method does not panic. + /// + /// # Safety + /// + /// If `self` describes the size and alignment of type that lacks static + /// shallow padding, unsafe code may assume that the result of this method + /// accurately reflects the size, alignment, and lack of static shallow + /// padding of that type. + const fn assume_shallow_unpadded(self) -> Self { + Self { statically_shallow_unpadded: true, ..self } + } + + /// Constructs a `DstLayout` for a zero-sized type with `repr_align` + /// alignment (or 1). If `repr_align` is provided, then it must be a power + /// of two. + /// + /// # Panics + /// + /// This function panics if the supplied `repr_align` is not a power of two. + /// + /// # Safety + /// + /// Unsafe code may assume that the contract of this function is satisfied. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn new_zst(repr_align: Option) -> DstLayout { + let align = match repr_align { + Some(align) => align, + None => Self::MIN_ALIGN, + }; + + const_assert!(align.get().is_power_of_two()); + + DstLayout { + align, + size_info: SizeInfo::Sized { size: 0 }, + statically_shallow_unpadded: true, + } + } + + /// Constructs a `DstLayout` which describes `T` and assumes `T` may contain + /// padding. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `T`. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn for_type() -> DstLayout { + // SAFETY: `align` is correct by construction. `T: Sized`, and so it is + // sound to initialize `size_info` to `SizeInfo::Sized { size }`; the + // `size` field is also correct by construction. `unpadded` can safely + // default to `false`. + DstLayout { + align: match NonZeroUsize::new(mem::align_of::()) { + Some(align) => align, + None => const_unreachable!(), + }, + size_info: SizeInfo::Sized { size: mem::size_of::() }, + statically_shallow_unpadded: false, + } + } + + /// Constructs a `DstLayout` which describes a `T` that does not contain + /// padding. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `T`. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn for_unpadded_type() -> DstLayout { + Self::for_type::().assume_shallow_unpadded() + } + + /// Constructs a `DstLayout` which describes `[T]`. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `[T]`. + pub(crate) const fn for_slice() -> DstLayout { + // SAFETY: The alignment of a slice is equal to the alignment of its + // element type, and so `align` is initialized correctly. + // + // Since this is just a slice type, there is no offset between the + // beginning of the type and the beginning of the slice, so it is + // correct to set `offset: 0`. The `elem_size` is correct by + // construction. Since `[T]` is a (degenerate case of a) slice DST, it + // is correct to initialize `size_info` to `SizeInfo::SliceDst`. + DstLayout { + align: match NonZeroUsize::new(mem::align_of::()) { + Some(align) => align, + None => const_unreachable!(), + }, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 0, + elem_size: mem::size_of::(), + }), + statically_shallow_unpadded: true, + } + } + + /// Constructs a complete `DstLayout` reflecting a `repr(C)` struct with the + /// given alignment modifiers and fields. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// For any definition of a `repr(C)` struct, if this method is invoked with + /// alignment modifiers and fields corresponding to that definition, the + /// resulting `DstLayout` will correctly encode the layout of that struct. + /// + /// We make no guarantees to the behavior of this method when it is invoked + /// with arguments that cannot correspond to a valid `repr(C)` struct. + #[must_use] + #[inline] + pub const fn for_repr_c_struct( + repr_align: Option, + repr_packed: Option, + fields: &[DstLayout], + ) -> DstLayout { + let mut layout = DstLayout::new_zst(repr_align); + + let mut i = 0; + #[allow(clippy::arithmetic_side_effects)] + while i < fields.len() { + #[allow(clippy::indexing_slicing)] + let field = fields[i]; + layout = layout.extend(field, repr_packed); + i += 1; + } + + layout = layout.pad_to_align(); + + // SAFETY: `layout` accurately describes the layout of a `repr(C)` + // struct with `repr_align` or `repr_packed` alignment modifications and + // the given `fields`. The `layout` is constructed using a sequence of + // invocations of `DstLayout::{new_zst,extend,pad_to_align}`. The + // documentation of these items vows that invocations in this manner + // will accurately describe a type, so long as: + // + // - that type is `repr(C)`, + // - its fields are enumerated in the order they appear, + // - the presence of `repr_align` and `repr_packed` are correctly accounted for. + // + // We respect all three of these preconditions above. + layout + } + + /// Like `Layout::extend`, this creates a layout that describes a record + /// whose layout consists of `self` followed by `next` that includes the + /// necessary inter-field padding, but not any trailing padding. + /// + /// In order to match the layout of a `#[repr(C)]` struct, this method + /// should be invoked for each field in declaration order. To add trailing + /// padding, call `DstLayout::pad_to_align` after extending the layout for + /// all fields. If `self` corresponds to a type marked with + /// `repr(packed(N))`, then `repr_packed` should be set to `Some(N)`, + /// otherwise `None`. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// If a (potentially hypothetical) valid `repr(C)` Rust type begins with + /// fields whose layout are `self`, and those fields are immediately + /// followed by a field whose layout is `field`, then unsafe code may rely + /// on `self.extend(field, repr_packed)` producing a layout that correctly + /// encompasses those two components. + /// + /// We make no guarantees to the behavior of this method if these fragments + /// cannot appear in a valid Rust type (e.g., the concatenation of the + /// layouts would lead to a size larger than `isize::MAX`). + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn extend(self, field: DstLayout, repr_packed: Option) -> Self { + use util::{max, min, padding_needed_for}; + + // If `repr_packed` is `None`, there are no alignment constraints, and + // the value can be defaulted to `THEORETICAL_MAX_ALIGN`. + let max_align = match repr_packed { + Some(max_align) => max_align, + None => Self::THEORETICAL_MAX_ALIGN, + }; + + const_assert!(max_align.get().is_power_of_two()); + + // We use Kani to prove that this method is robust to future increases + // in Rust's maximum allowed alignment. However, if such a change ever + // actually occurs, we'd like to be notified via assertion failures. + #[cfg(not(kani))] + { + const_debug_assert!(self.align.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + const_debug_assert!(field.align.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + if let Some(repr_packed) = repr_packed { + const_debug_assert!(repr_packed.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + } + } + + // The field's alignment is clamped by `repr_packed` (i.e., the + // `repr(packed(N))` attribute, if any) [1]. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // The alignments of each field, for the purpose of positioning + // fields, is the smaller of the specified alignment and the alignment + // of the field's type. + let field_align = min(field.align, max_align); + + // The struct's alignment is the maximum of its previous alignment and + // `field_align`. + let align = max(self.align, field_align); + + let (interfield_padding, size_info) = match self.size_info { + // If the layout is already a DST, we panic; DSTs cannot be extended + // with additional fields. + SizeInfo::SliceDst(..) => const_panic!("Cannot extend a DST with additional fields."), + + SizeInfo::Sized { size: preceding_size } => { + // Compute the minimum amount of inter-field padding needed to + // satisfy the field's alignment, and offset of the trailing + // field. [1] + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // Inter-field padding is guaranteed to be the minimum + // required in order to satisfy each field's (possibly + // altered) alignment. + let padding = padding_needed_for(preceding_size, field_align); + + // This will not panic (and is proven to not panic, with Kani) + // if the layout components can correspond to a leading layout + // fragment of a valid Rust type, but may panic otherwise (e.g., + // combining or aligning the components would create a size + // exceeding `isize::MAX`). + let offset = match preceding_size.checked_add(padding) { + Some(offset) => offset, + None => const_panic!("Adding padding to `self`'s size overflows `usize`."), + }; + + ( + padding, + match field.size_info { + SizeInfo::Sized { size: field_size } => { + // If the trailing field is sized, the resulting layout + // will be sized. Its size will be the sum of the + // preceding layout, the size of the new field, and the + // size of inter-field padding between the two. + // + // This will not panic (and is proven with Kani to not + // panic) if the layout components can correspond to a + // leading layout fragment of a valid Rust type, but may + // panic otherwise (e.g., combining or aligning the + // components would create a size exceeding + // `usize::MAX`). + let size = match offset.checked_add(field_size) { + Some(size) => size, + None => const_panic!("`field` cannot be appended without the total size overflowing `usize`"), + }; + SizeInfo::Sized { size } + } + SizeInfo::SliceDst(TrailingSliceLayout { + offset: trailing_offset, + elem_size, + }) => { + // If the trailing field is dynamically sized, so too + // will the resulting layout. The offset of the trailing + // slice component is the sum of the offset of the + // trailing field and the trailing slice offset within + // that field. + // + // This will not panic (and is proven with Kani to not + // panic) if the layout components can correspond to a + // leading layout fragment of a valid Rust type, but may + // panic otherwise (e.g., combining or aligning the + // components would create a size exceeding + // `usize::MAX`). + let offset = match offset.checked_add(trailing_offset) { + Some(offset) => offset, + None => const_panic!("`field` cannot be appended without the total size overflowing `usize`"), + }; + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + }, + ) + } + }; + + let statically_shallow_unpadded = self.statically_shallow_unpadded + && field.statically_shallow_unpadded + && interfield_padding == 0; + + DstLayout { align, size_info, statically_shallow_unpadded } + } + + /// Like `Layout::pad_to_align`, this routine rounds the size of this layout + /// up to the nearest multiple of this type's alignment or `repr_packed` + /// (whichever is less). This method leaves DST layouts unchanged, since the + /// trailing padding of DSTs is computed at runtime. + /// + /// The accompanying boolean is `true` if the resulting composition of + /// fields necessitated static (as opposed to dynamic) padding; otherwise + /// `false`. + /// + /// In order to match the layout of a `#[repr(C)]` struct, this method + /// should be invoked after the invocations of [`DstLayout::extend`]. If + /// `self` corresponds to a type marked with `repr(packed(N))`, then + /// `repr_packed` should be set to `Some(N)`, otherwise `None`. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// If a (potentially hypothetical) valid `repr(C)` type begins with fields + /// whose layout are `self` followed only by zero or more bytes of trailing + /// padding (not included in `self`), then unsafe code may rely on + /// `self.pad_to_align(repr_packed)` producing a layout that correctly + /// encapsulates the layout of that type. + /// + /// We make no guarantees to the behavior of this method if `self` cannot + /// appear in a valid Rust type (e.g., because the addition of trailing + /// padding would lead to a size larger than `isize::MAX`). + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn pad_to_align(self) -> Self { + use util::padding_needed_for; + + let (static_padding, size_info) = match self.size_info { + // For sized layouts, we add the minimum amount of trailing padding + // needed to satisfy alignment. + SizeInfo::Sized { size: unpadded_size } => { + let padding = padding_needed_for(unpadded_size, self.align); + let size = match unpadded_size.checked_add(padding) { + Some(size) => size, + None => const_panic!("Adding padding caused size to overflow `usize`."), + }; + (padding, SizeInfo::Sized { size }) + } + // For DST layouts, trailing padding depends on the length of the + // trailing DST and is computed at runtime. This does not alter the + // offset or element size of the layout, so we leave `size_info` + // unchanged. + size_info @ SizeInfo::SliceDst(_) => (0, size_info), + }; + + let statically_shallow_unpadded = self.statically_shallow_unpadded && static_padding == 0; + + DstLayout { align: self.align, size_info, statically_shallow_unpadded } + } + + /// Produces `true` if `self` requires static padding; otherwise `false`. + #[must_use] + #[inline(always)] + pub const fn requires_static_padding(self) -> bool { + !self.statically_shallow_unpadded + } + + /// Produces `true` if there exists any metadata for which a type of layout + /// `self` would require dynamic trailing padding; otherwise `false`. + #[must_use] + #[inline(always)] + pub const fn requires_dynamic_padding(self) -> bool { + // A `% self.align.get()` cannot panic, since `align` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + match self.size_info { + SizeInfo::Sized { .. } => false, + SizeInfo::SliceDst(trailing_slice_layout) => { + // SAFETY: This predicate is formally proved sound by + // `proofs::prove_requires_dynamic_padding`. + trailing_slice_layout.offset % self.align.get() != 0 + || trailing_slice_layout.elem_size % self.align.get() != 0 + } + } + } + + /// Validates that a cast is sound from a layout perspective. + /// + /// Validates that the size and alignment requirements of a type with the + /// layout described in `self` would not be violated by performing a + /// `cast_type` cast from a pointer with address `addr` which refers to a + /// memory region of size `bytes_len`. + /// + /// If the cast is valid, `validate_cast_and_convert_metadata` returns + /// `(elems, split_at)`. If `self` describes a dynamically-sized type, then + /// `elems` is the maximum number of trailing slice elements for which a + /// cast would be valid (for sized types, `elem` is meaningless and should + /// be ignored). `split_at` is the index at which to split the memory region + /// in order for the prefix (suffix) to contain the result of the cast, and + /// in order for the remaining suffix (prefix) to contain the leftover + /// bytes. + /// + /// There are three conditions under which a cast can fail: + /// - The smallest possible value for the type is larger than the provided + /// memory region + /// - A prefix cast is requested, and `addr` does not satisfy `self`'s + /// alignment requirement + /// - A suffix cast is requested, and `addr + bytes_len` does not satisfy + /// `self`'s alignment requirement (as a consequence, since all instances + /// of the type are a multiple of its alignment, no size for the type will + /// result in a starting address which is properly aligned) + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may rely + /// on that assumption for the soundness of their code. In particular, the + /// caller may assume that, if `validate_cast_and_convert_metadata` returns + /// `Some((elems, split_at))`, then: + /// - A pointer to the type (for dynamically sized types, this includes + /// `elems` as its pointer metadata) describes an object of size `size <= + /// bytes_len` + /// - If this is a prefix cast: + /// - `addr` satisfies `self`'s alignment + /// - `size == split_at` + /// - If this is a suffix cast: + /// - `split_at == bytes_len - size` + /// - `addr + split_at` satisfies `self`'s alignment + /// + /// Note that this method does *not* ensure that a pointer constructed from + /// its return values will be a valid pointer. In particular, this method + /// does not reason about `isize` overflow, which is a requirement of many + /// Rust pointer APIs, and may at some point be determined to be a validity + /// invariant of pointer types themselves. This should never be a problem so + /// long as the arguments to this method are derived from a known-valid + /// pointer (e.g., one derived from a safe Rust reference), but it is + /// nonetheless the caller's responsibility to justify that pointer + /// arithmetic will not overflow based on a safety argument *other than* the + /// mere fact that this method returned successfully. + /// + /// # Panics + /// + /// `validate_cast_and_convert_metadata` will panic if `self` describes a + /// DST whose trailing slice element is zero-sized. + /// + /// If `addr + bytes_len` overflows `usize`, + /// `validate_cast_and_convert_metadata` may panic, or it may return + /// incorrect results. No guarantees are made about when + /// `validate_cast_and_convert_metadata` will panic. The caller should not + /// rely on `validate_cast_and_convert_metadata` panicking in any particular + /// condition, even if `debug_assertions` are enabled. + #[allow(unused)] + #[inline(always)] + pub(crate) const fn validate_cast_and_convert_metadata( + &self, + addr: usize, + bytes_len: usize, + cast_type: CastType, + ) -> Result<(usize, usize), MetadataCastError> { + // `debug_assert!`, but with `#[allow(clippy::arithmetic_side_effects)]`. + macro_rules! __const_debug_assert { + ($e:expr $(, $msg:expr)?) => { + const_debug_assert!({ + #[allow(clippy::arithmetic_side_effects)] + let e = $e; + e + } $(, $msg)?); + }; + } + + // Note that, in practice, `self` is always a compile-time constant. We + // do this check earlier than needed to ensure that we always panic as a + // result of bugs in the program (such as calling this function on an + // invalid type) instead of allowing this panic to be hidden if the cast + // would have failed anyway for runtime reasons (such as a too-small + // memory region). + // + // FIXME(#67): Once our MSRV is 1.65, use let-else: + // https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html#let-else-statements + let size_info = match self.size_info.try_to_nonzero_elem_size() { + Some(size_info) => size_info, + None => const_panic!("attempted to cast to slice type with zero-sized element"), + }; + + // Precondition + __const_debug_assert!( + addr.checked_add(bytes_len).is_some(), + "`addr` + `bytes_len` > usize::MAX" + ); + + // Alignment checks go in their own block to avoid introducing variables + // into the top-level scope. + { + // We check alignment for `addr` (for prefix casts) or `addr + + // bytes_len` (for suffix casts). For a prefix cast, the correctness + // of this check is trivial - `addr` is the address the object will + // live at. + // + // For a suffix cast, we know that all valid sizes for the type are + // a multiple of the alignment (and by safety precondition, we know + // `DstLayout` may only describe valid Rust types). Thus, a + // validly-sized instance which lives at a validly-aligned address + // must also end at a validly-aligned address. Thus, if the end + // address for a suffix cast (`addr + bytes_len`) is not aligned, + // then no valid start address will be aligned either. + let offset = match cast_type { + CastType::Prefix => 0, + CastType::Suffix => bytes_len, + }; + + // Addition is guaranteed not to overflow because `offset <= + // bytes_len`, and `addr + bytes_len <= usize::MAX` is a + // precondition of this method. Modulus is guaranteed not to divide + // by 0 because `align` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + if (addr + offset) % self.align.get() != 0 { + return Err(MetadataCastError::Alignment); + } + } + + let (elems, self_bytes) = match size_info { + SizeInfo::Sized { size } => { + if size > bytes_len { + return Err(MetadataCastError::Size); + } + (0, size) + } + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + // Calculate the maximum number of bytes that could be consumed + // - any number of bytes larger than this will either not be a + // multiple of the alignment, or will be larger than + // `bytes_len`. + let max_total_bytes = + util::round_down_to_next_multiple_of_alignment(bytes_len, self.align); + // Calculate the maximum number of bytes that could be consumed + // by the trailing slice. + // + // FIXME(#67): Once our MSRV is 1.65, use let-else: + // https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html#let-else-statements + let max_slice_and_padding_bytes = match max_total_bytes.checked_sub(offset) { + Some(max) => max, + // `bytes_len` too small even for 0 trailing slice elements. + None => return Err(MetadataCastError::Size), + }; + + // Calculate the number of elements that fit in + // `max_slice_and_padding_bytes`; any remaining bytes will be + // considered padding. + // + // Guaranteed not to divide by zero: `elem_size` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + let elems = max_slice_and_padding_bytes / elem_size.get(); + // Guaranteed not to overflow on multiplication: `usize::MAX >= + // max_slice_and_padding_bytes >= (max_slice_and_padding_bytes / + // elem_size) * elem_size`. + // + // Guaranteed not to overflow on addition: + // - max_slice_and_padding_bytes == max_total_bytes - offset + // - elems * elem_size <= max_slice_and_padding_bytes == max_total_bytes - offset + // - elems * elem_size + offset <= max_total_bytes <= usize::MAX + #[allow(clippy::arithmetic_side_effects)] + let without_padding = offset + elems * elem_size.get(); + // `self_bytes` is equal to the offset bytes plus the bytes + // consumed by the trailing slice plus any padding bytes + // required to satisfy the alignment. Note that we have computed + // the maximum number of trailing slice elements that could fit + // in `self_bytes`, so any padding is guaranteed to be less than + // the size of an extra element. + // + // Guaranteed not to overflow: + // - By previous comment: without_padding == elems * elem_size + + // offset <= max_total_bytes + // - By construction, `max_total_bytes` is a multiple of + // `self.align`. + // - At most, adding padding needed to round `without_padding` + // up to the next multiple of the alignment will bring + // `self_bytes` up to `max_total_bytes`. + #[allow(clippy::arithmetic_side_effects)] + let self_bytes = + without_padding + util::padding_needed_for(without_padding, self.align); + (elems, self_bytes) + } + }; + + __const_debug_assert!(self_bytes <= bytes_len); + + let split_at = match cast_type { + CastType::Prefix => self_bytes, + // Guaranteed not to underflow: + // - In the `Sized` branch, only returns `size` if `size <= + // bytes_len`. + // - In the `SliceDst` branch, calculates `self_bytes <= + // max_toatl_bytes`, which is upper-bounded by `bytes_len`. + #[allow(clippy::arithmetic_side_effects)] + CastType::Suffix => bytes_len - self_bytes, + }; + + Ok((elems, split_at)) + } +} + +pub(crate) use cast_from::CastFrom; +mod cast_from { + use crate::*; + + pub(crate) struct CastFrom { + _never: core::convert::Infallible, + _marker: PhantomData, + } + + // SAFETY: The implementation of `Project::project` preserves the address + // of the referent – it only modifies pointer metadata. + unsafe impl crate::pointer::cast::Cast for CastFrom + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + } + + // SAFETY: The implementation of `Project::project` preserves the size of + // the referent (see inline comments for a more detailed proof of this). + unsafe impl crate::pointer::cast::CastExact for CastFrom + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + } + + // SAFETY: `project` produces a pointer which refers to the same referent + // bytes as its input, or to a subset of them (see inline comments for a + // more detailed proof of this). It does this using provenance-preserving + // operations. + unsafe impl crate::pointer::cast::Project for CastFrom + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + /// # PME + /// + /// Generates a post-monomorphization error if it is not possible to + /// implement soundly. + // + // FIXME(#1817): Support Sized->Unsized and Unsized->Sized casts + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + /// The parameters required in order to perform a pointer cast from + /// `Src` to `Dst`. + /// + /// These are a compile-time function of the layouts of `Src` + /// and `Dst`. + /// + /// # Safety + /// + /// `Src`'s alignment must not be smaller than `Dst`'s alignment. + struct CastParams { + inner: CastParamsInner, + _src: PhantomData, + _dst: PhantomData, + } + + #[derive(Copy, Clone)] + enum CastParamsInner { + // At compile time (specifically, post-monomorphization time), + // we need to compute two things: + // - Whether, given *any* `*Src`, it is possible to construct a + // `*Dst` which addresses the same number of bytes (ie, + // whether, for any `Src` pointer metadata, there exists `Dst` + // pointer metadata that addresses the same number of bytes) + // - If this is possible, any information necessary to perform + // the `Src`->`Dst` metadata conversion at runtime. + // + // Assume that `Src` and `Dst` are slice DSTs, and define: + // - `S_OFF = Src::LAYOUT.size_info.offset` + // - `S_ELEM = Src::LAYOUT.size_info.elem_size` + // - `D_OFF = Dst::LAYOUT.size_info.offset` + // - `D_ELEM = Dst::LAYOUT.size_info.elem_size` + // + // We are trying to solve the following equation: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // At runtime, we will be attempting to compute `d_meta`, given + // `s_meta` (a runtime value) and all other parameters (which + // are compile-time values). We can solve like so: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // d_meta * D_ELEM = S_OFF - D_OFF + s_meta * S_ELEM + // + // d_meta = (S_OFF - D_OFF + s_meta * S_ELEM)/D_ELEM + // + // Since `d_meta` will be a `usize`, we need the right-hand side + // to be an integer, and this needs to hold for *any* value of + // `s_meta` (in order for our conversion to be infallible - ie, + // to not have to reject certain values of `s_meta` at runtime). + // This means that: + // + // - `s_meta * S_ELEM` must be a multiple of `D_ELEM` + // - Since this must hold for any value of `s_meta`, `S_ELEM` + // must be a multiple of `D_ELEM` + // - `S_OFF - D_OFF` must be a multiple of `D_ELEM` + // + // Thus, let `OFFSET_DELTA_ELEMS = (S_OFF - D_OFF)/D_ELEM` and + // `ELEM_MULTIPLE = S_ELEM/D_ELEM`. We can rewrite the above + // expression as: + // + // d_meta = (S_OFF - D_OFF + s_meta * S_ELEM)/D_ELEM + // + // d_meta = OFFSET_DELTA_ELEMS + s_meta * ELEM_MULTIPLE + // + // Thus, we just need to compute the following and confirm that + // they have integer solutions in order to both a) determine + // whether infallible `Src` -> `Dst` casts are possible and, b) + // pre-compute the parameters necessary to perform those casts + // at runtime. These parameters are encapsulated in + // `CastParams`, which acts as a witness that such infallible + // casts are possible. + /// The parameters required in order to perform an + /// unsized-to-unsized pointer cast from `Src` to `Dst` as + /// described above. + /// + /// # Safety + /// + /// `Src` and `Dst` must both be slice DSTs. + /// + /// `offset_delta_elems` and `elem_multiple` must be valid as + /// described above. + UnsizedToUnsized { offset_delta_elems: usize, elem_multiple: usize }, + + /// The metadata of a `Dst` which has the same size as `Src: + /// Sized`. + /// + /// # Safety + /// + /// `Src: Sized` and `Dst` must be a slice DST. + /// + /// A raw `Dst` pointer with metadata `dst_meta` must address + /// `size_of::()` bytes. + SizedToUnsized { dst_meta: usize }, + + /// The metadata of a `Dst` which has the same size as `Src: + /// Sized`. + /// + /// # Safety + /// + /// `Src` and `Dst` must both be `Sized` and `size_of::() + /// == size_of::()`. + SizedToSized, + } + + impl Copy for CastParams {} + impl Clone for CastParams { + fn clone(&self) -> Self { + *self + } + } + + impl CastParams { + const fn try_compute( + src: &DstLayout, + dst: &DstLayout, + ) -> Option> { + if src.align.get() < dst.align.get() { + return None; + } + + let inner = match (src.size_info, dst.size_info) { + ( + SizeInfo::Sized { size: src_size }, + SizeInfo::Sized { size: dst_size }, + ) => { + if src_size != dst_size { + return None; + } + + // SAFETY: We checked above that `src_size == + // dst_size`. + CastParamsInner::SizedToSized + } + (SizeInfo::Sized { size: src_size }, SizeInfo::SliceDst(dst)) => { + let offset_delta = if let Some(od) = src_size.checked_sub(dst.offset) { + od + } else { + return None; + }; + + let dst_elem_size = if let Some(e) = NonZeroUsize::new(dst.elem_size) { + e + } else { + return None; + }; + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let delta_mod_other_elem = offset_delta % dst_elem_size.get(); + + if delta_mod_other_elem != 0 { + return None; + } + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let dst_meta = offset_delta / dst_elem_size.get(); + + // SAFETY: The preceding math ensures that a `Dst` + // with `dst_meta` addresses `src_size` bytes. + CastParamsInner::SizedToUnsized { dst_meta } + } + (SizeInfo::SliceDst(src), SizeInfo::SliceDst(dst)) => { + let offset_delta = if let Some(od) = src.offset.checked_sub(dst.offset) + { + od + } else { + return None; + }; + + let dst_elem_size = if let Some(e) = NonZeroUsize::new(dst.elem_size) { + e + } else { + return None; + }; + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let delta_mod_other_elem = offset_delta % dst_elem_size.get(); + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let elem_remainder = src.elem_size % dst_elem_size.get(); + + if delta_mod_other_elem != 0 + || src.elem_size < dst.elem_size + || elem_remainder != 0 + { + return None; + } + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let offset_delta_elems = offset_delta / dst_elem_size.get(); + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let elem_multiple = src.elem_size / dst_elem_size.get(); + + CastParamsInner::UnsizedToUnsized { + // SAFETY: We checked above that this is an exact ratio. + offset_delta_elems, + // SAFETY: We checked above that this is an exact ratio. + elem_multiple, + } + } + _ => return None, + }; + + // SAFETY: We checked above that `src.align >= dst.align`. + Some(CastParams { inner, _src: PhantomData, _dst: PhantomData }) + } + } + + impl CastParams { + /// # Safety + /// + /// `src_meta` describes a `Src` whose size is no larger than + /// `isize::MAX`. + /// + /// The returned metadata describes a `Dst` of the same size as + /// the original `Src`. + #[inline(always)] + unsafe fn cast_metadata( + self, + src_meta: Src::PointerMetadata, + ) -> Dst::PointerMetadata { + #[allow(unused)] + use crate::util::polyfills::*; + + let dst_meta = match self.inner { + CastParamsInner::UnsizedToUnsized { offset_delta_elems, elem_multiple } => { + let src_meta = src_meta.to_elem_count(); + #[allow( + unstable_name_collisions, + clippy::multiple_unsafe_ops_per_block + )] + // SAFETY: `self` is a witness that the following + // equation holds: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // Since the caller promises that `src_meta` is + // valid `Src` metadata, this math will not + // overflow, and the returned value will describe a + // `Dst` of the same size. + unsafe { + offset_delta_elems + .unchecked_add(src_meta.unchecked_mul(elem_multiple)) + } + } + CastParamsInner::SizedToUnsized { dst_meta } => dst_meta, + CastParamsInner::SizedToSized => 0, + }; + Dst::PointerMetadata::from_elem_count(dst_meta) + } + } + + trait Params { + const CAST_PARAMS: CastParams; + } + + impl Params for Dst + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + const CAST_PARAMS: CastParams = + match CastParams::try_compute(&Src::LAYOUT, &Dst::LAYOUT) { + Some(params) => params, + None => const_panic!( + "cannot `transmute_ref!` or `transmute_mut!` between incompatible types" + ), + }; + } + + let src_meta = ::pointer_to_metadata(src.as_ptr()); + let params = >::CAST_PARAMS; + + // SAFETY: `src: PtrInner` guarantees that `src`'s referent is zero + // bytes or lives in a single allocation, which means that it is no + // larger than `isize::MAX` bytes [1]. + // + // [1] https://doc.rust-lang.org/1.92.0/std/ptr/index.html#allocation + let dst_meta = unsafe { params.cast_metadata(src_meta) }; + + ::raw_from_ptr_len(src.as_non_null().cast(), dst_meta).as_ptr() + } + } +} + +// FIXME(#67): For some reason, on our MSRV toolchain, this `allow` isn't +// enforced despite having `#![allow(unknown_lints)]` at the crate root, but +// putting it here works. Once our MSRV is high enough that this bug has been +// fixed, remove this `allow`. +#[allow(unknown_lints)] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dst_layout_for_slice() { + let layout = DstLayout::for_slice::(); + match layout.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + assert_eq!(offset, 0); + assert_eq!(elem_size, 4); + } + _ => panic!("Expected SliceDst"), + } + assert_eq!(layout.align.get(), 4); + } + + /// Tests of when a sized `DstLayout` is extended with a sized field. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_dst_layout_extend_sized_with_sized() { + // This macro constructs a layout corresponding to a `u8` and extends it + // with a zero-sized trailing field of given alignment `n`. The macro + // tests that the resulting layout has both size and alignment `min(n, + // P)` for all valid values of `repr(packed(P))`. + macro_rules! test_align_is_size { + ($n:expr) => { + let base = DstLayout::for_type::(); + let trailing_field = DstLayout::for_type::>(); + + let packs = + core::iter::once(None).chain((0..29).map(|p| NonZeroUsize::new(2usize.pow(p)))); + + for pack in packs { + let composite = base.extend(trailing_field, pack); + let max_align = pack.unwrap_or(DstLayout::CURRENT_MAX_ALIGN); + let align = $n.min(max_align.get()); + assert_eq!( + composite, + DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::Sized { size: align }, + statically_shallow_unpadded: false, + } + ) + } + }; + } + + test_align_is_size!(1); + test_align_is_size!(2); + test_align_is_size!(4); + test_align_is_size!(8); + test_align_is_size!(16); + test_align_is_size!(32); + test_align_is_size!(64); + test_align_is_size!(128); + test_align_is_size!(256); + test_align_is_size!(512); + test_align_is_size!(1024); + test_align_is_size!(2048); + test_align_is_size!(4096); + test_align_is_size!(8192); + test_align_is_size!(16384); + test_align_is_size!(32768); + test_align_is_size!(65536); + test_align_is_size!(131072); + test_align_is_size!(262144); + test_align_is_size!(524288); + test_align_is_size!(1048576); + test_align_is_size!(2097152); + test_align_is_size!(4194304); + test_align_is_size!(8388608); + test_align_is_size!(16777216); + test_align_is_size!(33554432); + test_align_is_size!(67108864); + test_align_is_size!(33554432); + test_align_is_size!(134217728); + test_align_is_size!(268435456); + } + + /// Tests of when a sized `DstLayout` is extended with a DST field. + #[test] + fn test_dst_layout_extend_sized_with_dst() { + // Test that for all combinations of real-world alignments and + // `repr_packed` values, that the extension of a sized `DstLayout`` with + // a DST field correctly computes the trailing offset in the composite + // layout. + + let aligns = (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()); + let packs = core::iter::once(None).chain(aligns.clone().map(Some)); + + for align in aligns { + for pack in packs.clone() { + let base = DstLayout::for_type::(); + let elem_size = 42; + let trailing_field_offset = 11; + + let trailing_field = DstLayout { + align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { elem_size, offset: 11 }), + statically_shallow_unpadded: false, + }; + + let composite = base.extend(trailing_field, pack); + + let max_align = pack.unwrap_or(DstLayout::CURRENT_MAX_ALIGN).get(); + + let align = align.get().min(max_align); + + assert_eq!( + composite, + DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + elem_size, + offset: align + trailing_field_offset, + }), + statically_shallow_unpadded: false, + } + ) + } + } + } + + /// Tests that calling `pad_to_align` on a sized `DstLayout` adds the + /// expected amount of trailing padding. + #[test] + fn test_dst_layout_pad_to_align_with_sized() { + // For all valid alignments `align`, construct a one-byte layout aligned + // to `align`, call `pad_to_align`, and assert that the size of the + // resulting layout is equal to `align`. + for align in (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()) { + let layout = DstLayout { + align, + size_info: SizeInfo::Sized { size: 1 }, + statically_shallow_unpadded: true, + }; + + assert_eq!( + layout.pad_to_align(), + DstLayout { + align, + size_info: SizeInfo::Sized { size: align.get() }, + statically_shallow_unpadded: align.get() == 1 + } + ); + } + + // Test explicitly-provided combinations of unpadded and padded + // counterparts. + + macro_rules! test { + (unpadded { size: $unpadded_size:expr, align: $unpadded_align:expr } + => padded { size: $padded_size:expr, align: $padded_align:expr }) => { + let unpadded = DstLayout { + align: NonZeroUsize::new($unpadded_align).unwrap(), + size_info: SizeInfo::Sized { size: $unpadded_size }, + statically_shallow_unpadded: false, + }; + let padded = unpadded.pad_to_align(); + + assert_eq!( + padded, + DstLayout { + align: NonZeroUsize::new($padded_align).unwrap(), + size_info: SizeInfo::Sized { size: $padded_size }, + statically_shallow_unpadded: false, + } + ); + }; + } + + test!(unpadded { size: 0, align: 4 } => padded { size: 0, align: 4 }); + test!(unpadded { size: 1, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 2, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 3, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 4, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 5, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 6, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 7, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 8, align: 4 } => padded { size: 8, align: 4 }); + + let current_max_align = DstLayout::CURRENT_MAX_ALIGN.get(); + + test!(unpadded { size: 1, align: current_max_align } + => padded { size: current_max_align, align: current_max_align }); + + test!(unpadded { size: current_max_align + 1, align: current_max_align } + => padded { size: current_max_align * 2, align: current_max_align }); + } + + /// Tests that calling `pad_to_align` on a DST `DstLayout` is a no-op. + #[test] + fn test_dst_layout_pad_to_align_with_dst() { + for align in (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()) { + for offset in 0..10 { + for elem_size in 0..10 { + let layout = DstLayout { + align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }), + statically_shallow_unpadded: false, + }; + assert_eq!(layout.pad_to_align(), layout); + } + } + } + } + + // This test takes a long time when running under Miri, so we skip it in + // that case. This is acceptable because this is a logic test that doesn't + // attempt to expose UB. + #[test] + #[cfg_attr(miri, ignore)] + fn test_validate_cast_and_convert_metadata() { + #[allow(non_local_definitions)] + impl From for SizeInfo { + fn from(size: usize) -> SizeInfo { + SizeInfo::Sized { size } + } + } + + #[allow(non_local_definitions)] + impl From<(usize, usize)> for SizeInfo { + fn from((offset, elem_size): (usize, usize)) -> SizeInfo { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + } + + fn layout>(s: S, align: usize) -> DstLayout { + DstLayout { + size_info: s.into(), + align: NonZeroUsize::new(align).unwrap(), + statically_shallow_unpadded: false, + } + } + + /// This macro accepts arguments in the form of: + /// + /// layout(_, _).validate(_, _, _), Ok(Some((_, _))) + /// | | | | | | | + /// size ---------+ | | | | | | + /// align -----------+ | | | | | + /// addr ------------------------+ | | | | + /// bytes_len ----------------------+ | | | + /// cast_type -------------------------+ | | + /// elems ------------------------------------------+ | + /// split_at ------------------------------------------+ + /// + /// `.validate` is shorthand for `.validate_cast_and_convert_metadata` + /// for brevity. + /// + /// Each argument can either be an iterator or a wildcard. Each + /// wildcarded variable is implicitly replaced by an iterator over a + /// representative sample of values for that variable. Each `test!` + /// invocation iterates over every combination of values provided by + /// each variable's iterator (ie, the cartesian product) and validates + /// that the results are expected. + /// + /// The final argument uses the same syntax, but it has a different + /// meaning: + /// - If it is `Ok(pat)`, then the pattern `pat` is supplied to + /// a matching assert to validate the computed result for each + /// combination of input values. + /// - If it is `Err(Some(msg) | None)`, then `test!` validates that the + /// call to `validate_cast_and_convert_metadata` panics with the given + /// panic message or, if the current Rust toolchain version is too + /// early to support panicking in `const fn`s, panics with *some* + /// message. In the latter case, the `const_panic!` macro is used, + /// which emits code which causes a non-panicking error at const eval + /// time, but which does panic when invoked at runtime. Thus, it is + /// merely difficult to predict the *value* of this panic. We deem + /// that testing against the real panic strings on stable and nightly + /// toolchains is enough to ensure correctness. + /// + /// Note that the meta-variables that match these variables have the + /// `tt` type, and some valid expressions are not valid `tt`s (such as + /// `a..b`). In this case, wrap the expression in parentheses, and it + /// will become valid `tt`. + macro_rules! test { + ( + layout($size:tt, $align:tt) + .validate($addr:tt, $bytes_len:tt, $cast_type:tt), $expect:pat $(,)? + ) => { + itertools::iproduct!( + test!(@generate_size $size), + test!(@generate_align $align), + test!(@generate_usize $addr), + test!(@generate_usize $bytes_len), + test!(@generate_cast_type $cast_type) + ).for_each(|(size_info, align, addr, bytes_len, cast_type)| { + // Temporarily disable the panic hook installed by the test + // harness. If we don't do this, all panic messages will be + // kept in an internal log. On its own, this isn't a + // problem, but if a non-caught panic ever happens (ie, in + // code later in this test not in this macro), all of the + // previously-buffered messages will be dumped, hiding the + // real culprit. + let previous_hook = std::panic::take_hook(); + // I don't understand why, but this seems to be required in + // addition to the previous line. + std::panic::set_hook(Box::new(|_| {})); + let actual = std::panic::catch_unwind(|| { + layout(size_info, align).validate_cast_and_convert_metadata(addr, bytes_len, cast_type) + }).map_err(|d| { + let msg = d.downcast::<&'static str>().ok().map(|s| *s.as_ref()); + assert!(msg.is_some() || cfg!(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0), "non-string panic messages are not permitted when usage of panic in const fn is enabled"); + msg + }); + std::panic::set_hook(previous_hook); + + assert!( + matches!(actual, $expect), + "layout({:?}, {}).validate_cast_and_convert_metadata({}, {}, {:?})" ,size_info, align, addr, bytes_len, cast_type + ); + }); + }; + (@generate_usize _) => { 0..8 }; + // Generate sizes for both Sized and !Sized types. + (@generate_size _) => { + test!(@generate_size (_)).chain(test!(@generate_size (_, _))) + }; + // Generate sizes for both Sized and !Sized types by chaining + // specified iterators for each. + (@generate_size ($sized_sizes:tt | $unsized_sizes:tt)) => { + test!(@generate_size ($sized_sizes)).chain(test!(@generate_size $unsized_sizes)) + }; + // Generate sizes for Sized types. + (@generate_size (_)) => { test!(@generate_size (0..8)) }; + (@generate_size ($sizes:expr)) => { $sizes.into_iter().map(Into::::into) }; + // Generate sizes for !Sized types. + (@generate_size ($min_sizes:tt, $elem_sizes:tt)) => { + itertools::iproduct!( + test!(@generate_min_size $min_sizes), + test!(@generate_elem_size $elem_sizes) + ).map(Into::::into) + }; + (@generate_fixed_size _) => { (0..8).into_iter().map(Into::::into) }; + (@generate_min_size _) => { 0..8 }; + (@generate_elem_size _) => { 1..8 }; + (@generate_align _) => { [1, 2, 4, 8, 16] }; + (@generate_opt_usize _) => { [None].into_iter().chain((0..8).map(Some).into_iter()) }; + (@generate_cast_type _) => { [CastType::Prefix, CastType::Suffix] }; + (@generate_cast_type $variant:ident) => { [CastType::$variant] }; + // Some expressions need to be wrapped in parentheses in order to be + // valid `tt`s (required by the top match pattern). See the comment + // below for more details. This arm removes these parentheses to + // avoid generating an `unused_parens` warning. + (@$_:ident ($vals:expr)) => { $vals }; + (@$_:ident $vals:expr) => { $vals }; + } + + const EVENS: [usize; 8] = [0, 2, 4, 6, 8, 10, 12, 14]; + const ODDS: [usize; 8] = [1, 3, 5, 7, 9, 11, 13, 15]; + + // base_size is too big for the memory region. + test!( + layout(((1..8) | ((1..8), (1..8))), _).validate([0], [0], _), + Ok(Err(MetadataCastError::Size)) + ); + test!( + layout(((2..8) | ((2..8), (2..8))), _).validate([0], [1], Prefix), + Ok(Err(MetadataCastError::Size)) + ); + test!( + layout(((2..8) | ((2..8), (2..8))), _).validate([0x1000_0000 - 1], [1], Suffix), + Ok(Err(MetadataCastError::Size)) + ); + + // addr is unaligned for prefix cast + test!(layout(_, [2]).validate(ODDS, _, Prefix), Ok(Err(MetadataCastError::Alignment))); + test!(layout(_, [2]).validate(ODDS, _, Prefix), Ok(Err(MetadataCastError::Alignment))); + + // addr is aligned, but end of buffer is unaligned for suffix cast + test!(layout(_, [2]).validate(EVENS, ODDS, Suffix), Ok(Err(MetadataCastError::Alignment))); + test!(layout(_, [2]).validate(EVENS, ODDS, Suffix), Ok(Err(MetadataCastError::Alignment))); + + // Unfortunately, these constants cannot easily be used in the + // implementation of `validate_cast_and_convert_metadata`, since + // `panic!` consumes a string literal, not an expression. + // + // It's important that these messages be in a separate module. If they + // were at the function's top level, we'd pass them to `test!` as, e.g., + // `Err(TRAILING)`, which would run into a subtle Rust footgun - the + // `TRAILING` identifier would be treated as a pattern to match rather + // than a value to check for equality. + mod msgs { + pub(super) const TRAILING: &str = + "attempted to cast to slice type with zero-sized element"; + pub(super) const OVERFLOW: &str = "`addr` + `bytes_len` > usize::MAX"; + } + + // casts with ZST trailing element types are unsupported + test!(layout((_, [0]), _).validate(_, _, _), Err(Some(msgs::TRAILING) | None),); + + // addr + bytes_len must not overflow usize + test!(layout(_, _).validate([usize::MAX], (1..100), _), Err(Some(msgs::OVERFLOW) | None)); + test!(layout(_, _).validate((1..100), [usize::MAX], _), Err(Some(msgs::OVERFLOW) | None)); + test!( + layout(_, _).validate( + [usize::MAX / 2 + 1, usize::MAX], + [usize::MAX / 2 + 1, usize::MAX], + _ + ), + Err(Some(msgs::OVERFLOW) | None) + ); + + // Validates that `validate_cast_and_convert_metadata` satisfies its own + // documented safety postconditions, and also a few other properties + // that aren't documented but we want to guarantee anyway. + fn validate_behavior( + (layout, addr, bytes_len, cast_type): (DstLayout, usize, usize, CastType), + ) { + if let Ok((elems, split_at)) = + layout.validate_cast_and_convert_metadata(addr, bytes_len, cast_type) + { + let (size_info, align) = (layout.size_info, layout.align); + let debug_str = format!( + "layout({:?}, {}).validate_cast_and_convert_metadata({}, {}, {:?}) => ({}, {})", + size_info, align, addr, bytes_len, cast_type, elems, split_at + ); + + // If this is a sized type (no trailing slice), then `elems` is + // meaningless, but in practice we set it to 0. Callers are not + // allowed to rely on this, but a lot of math is nicer if + // they're able to, and some callers might accidentally do that. + let sized = matches!(layout.size_info, SizeInfo::Sized { .. }); + assert!(!(sized && elems != 0), "{}", debug_str); + + let resulting_size = match layout.size_info { + SizeInfo::Sized { size } => size, + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + let padded_size = |elems| { + let without_padding = offset + elems * elem_size; + without_padding + util::padding_needed_for(without_padding, align) + }; + + let resulting_size = padded_size(elems); + // Test that `validate_cast_and_convert_metadata` + // computed the largest possible value that fits in the + // given range. + assert!(padded_size(elems + 1) > bytes_len, "{}", debug_str); + resulting_size + } + }; + + // Test safety postconditions guaranteed by + // `validate_cast_and_convert_metadata`. + assert!(resulting_size <= bytes_len, "{}", debug_str); + match cast_type { + CastType::Prefix => { + assert_eq!(addr % align, 0, "{}", debug_str); + assert_eq!(resulting_size, split_at, "{}", debug_str); + } + CastType::Suffix => { + assert_eq!(split_at, bytes_len - resulting_size, "{}", debug_str); + assert_eq!((addr + split_at) % align, 0, "{}", debug_str); + } + } + } else { + let min_size = match layout.size_info { + SizeInfo::Sized { size } => size, + SizeInfo::SliceDst(TrailingSliceLayout { offset, .. }) => { + offset + util::padding_needed_for(offset, layout.align) + } + }; + + // If a cast is invalid, it is either because... + // 1. there are insufficient bytes at the given region for type: + let insufficient_bytes = bytes_len < min_size; + // 2. performing the cast would misalign type: + let base = match cast_type { + CastType::Prefix => 0, + CastType::Suffix => bytes_len, + }; + let misaligned = (base + addr) % layout.align != 0; + + assert!(insufficient_bytes || misaligned); + } + } + + let sizes = 0..8; + let elem_sizes = 1..8; + let size_infos = sizes + .clone() + .map(Into::::into) + .chain(itertools::iproduct!(sizes, elem_sizes).map(Into::::into)); + let layouts = itertools::iproduct!(size_infos, [1, 2, 4, 8, 16, 32]) + .filter(|(size_info, align)| !matches!(size_info, SizeInfo::Sized { size } if size % align != 0)) + .map(|(size_info, align)| layout(size_info, align)); + itertools::iproduct!(layouts, 0..8, 0..8, [CastType::Prefix, CastType::Suffix]) + .for_each(validate_behavior); + } + + #[test] + #[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] + fn test_validate_rust_layout() { + use core::{ + convert::TryInto as _, + ptr::{self, NonNull}, + }; + + use crate::util::testutil::*; + + // This test synthesizes pointers with various metadata and uses Rust's + // built-in APIs to confirm that Rust makes decisions about type layout + // which are consistent with what we believe is guaranteed by the + // language. If this test fails, it doesn't just mean our code is wrong + // - it means we're misunderstanding the language's guarantees. + + #[derive(Debug)] + struct MacroArgs { + offset: usize, + align: NonZeroUsize, + elem_size: Option, + } + + /// # Safety + /// + /// `test` promises to only call `addr_of_slice_field` on a `NonNull` + /// which points to a valid `T`. + /// + /// `with_elems` must produce a pointer which points to a valid `T`. + fn test NonNull>( + args: MacroArgs, + with_elems: W, + addr_of_slice_field: Option) -> NonNull>, + ) { + let dst = args.elem_size.is_some(); + let layout = { + let size_info = match args.elem_size { + Some(elem_size) => { + SizeInfo::SliceDst(TrailingSliceLayout { offset: args.offset, elem_size }) + } + None => SizeInfo::Sized { + // Rust only supports types whose sizes are a multiple + // of their alignment. If the macro created a type like + // this: + // + // #[repr(C, align(2))] + // struct Foo([u8; 1]); + // + // ...then Rust will automatically round the type's size + // up to 2. + size: args.offset + util::padding_needed_for(args.offset, args.align), + }, + }; + DstLayout { size_info, align: args.align, statically_shallow_unpadded: false } + }; + + for elems in 0..128 { + let ptr = with_elems(elems); + + if let Some(addr_of_slice_field) = addr_of_slice_field { + let slc_field_ptr = addr_of_slice_field(ptr).as_ptr(); + // SAFETY: Both `slc_field_ptr` and `ptr` are pointers to + // the same valid Rust object. + // Work around https://github.com/rust-lang/rust-clippy/issues/12280 + let offset: usize = + unsafe { slc_field_ptr.byte_offset_from(ptr.as_ptr()).try_into().unwrap() }; + assert_eq!(offset, args.offset); + } + + // SAFETY: `ptr` points to a valid `T`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + let (size, align) = unsafe { + (mem::size_of_val_raw(ptr.as_ptr()), mem::align_of_val_raw(ptr.as_ptr())) + }; + + // Avoid expensive allocation when running under Miri. + let assert_msg = if !cfg!(miri) { + format!("\n{:?}\nsize:{}, align:{}", args, size, align) + } else { + String::new() + }; + + let without_padding = + args.offset + args.elem_size.map(|elem_size| elems * elem_size).unwrap_or(0); + assert!(size >= without_padding, "{}", assert_msg); + assert_eq!(align, args.align.get(), "{}", assert_msg); + + // This encodes the most important part of the test: our + // understanding of how Rust determines the layout of repr(C) + // types. Sized repr(C) types are trivial, but DST types have + // some subtlety. Note that: + // - For sized types, `without_padding` is just the size of the + // type that we constructed for `Foo`. Since we may have + // requested a larger alignment, `Foo` may actually be larger + // than this, hence `padding_needed_for`. + // - For unsized types, `without_padding` is dynamically + // computed from the offset, the element size, and element + // count. We expect that the size of the object should be + // `offset + elem_size * elems` rounded up to the next + // alignment. + let expected_size = + without_padding + util::padding_needed_for(without_padding, args.align); + assert_eq!(expected_size, size, "{}", assert_msg); + + // For zero-sized element types, + // `validate_cast_and_convert_metadata` just panics, so we skip + // testing those types. + if args.elem_size.map(|elem_size| elem_size > 0).unwrap_or(true) { + let addr = ptr.addr().get(); + let (got_elems, got_split_at) = layout + .validate_cast_and_convert_metadata(addr, size, CastType::Prefix) + .unwrap(); + // Avoid expensive allocation when running under Miri. + let assert_msg = if !cfg!(miri) { + format!( + "{}\nvalidate_cast_and_convert_metadata({}, {})", + assert_msg, addr, size, + ) + } else { + String::new() + }; + assert_eq!(got_split_at, size, "{}", assert_msg); + if dst { + assert!(got_elems >= elems, "{}", assert_msg); + if got_elems != elems { + // If `validate_cast_and_convert_metadata` + // returned more elements than `elems`, that + // means that `elems` is not the maximum number + // of elements that can fit in `size` - in other + // words, there is enough padding at the end of + // the value to fit at least one more element. + // If we use this metadata to synthesize a + // pointer, despite having a different element + // count, we still expect it to have the same + // size. + let got_ptr = with_elems(got_elems); + // SAFETY: `got_ptr` is a pointer to a valid `T`. + let size_of_got_ptr = unsafe { mem::size_of_val_raw(got_ptr.as_ptr()) }; + assert_eq!(size_of_got_ptr, size, "{}", assert_msg); + } + } else { + // For sized casts, the returned element value is + // technically meaningless, and we don't guarantee any + // particular value. In practice, it's always zero. + assert_eq!(got_elems, 0, "{}", assert_msg) + } + } + } + } + + macro_rules! validate_against_rust { + ($offset:literal, $align:literal $(, $elem_size:literal)?) => {{ + #[repr(C, align($align))] + struct Foo([u8; $offset]$(, [[u8; $elem_size]])?); + + let args = MacroArgs { + offset: $offset, + align: $align.try_into().unwrap(), + elem_size: { + #[allow(unused)] + let ret = None::; + $(let ret = Some($elem_size);)? + ret + } + }; + + #[repr(C, align($align))] + struct FooAlign; + // Create an aligned buffer to use in order to synthesize + // pointers to `Foo`. We don't ever load values from these + // pointers - we just do arithmetic on them - so having a "real" + // block of memory as opposed to a validly-aligned-but-dangling + // pointer is only necessary to make Miri happy since we run it + // with "strict provenance" checking enabled. + let aligned_buf = Align::<_, FooAlign>::new([0u8; 1024]); + let with_elems = |elems| { + let slc = NonNull::slice_from_raw_parts(NonNull::from(&aligned_buf.t), elems); + #[allow(clippy::as_conversions)] + NonNull::new(slc.as_ptr() as *mut Foo).unwrap() + }; + let addr_of_slice_field = { + #[allow(unused)] + let f = None::) -> NonNull>; + $( + // SAFETY: `test` promises to only call `f` with a `ptr` + // to a valid `Foo`. + let f: Option) -> NonNull> = Some(|ptr: NonNull| unsafe { + NonNull::new(ptr::addr_of_mut!((*ptr.as_ptr()).1)).unwrap().cast::() + }); + let _ = $elem_size; + )? + f + }; + + test::(args, with_elems, addr_of_slice_field); + }}; + } + + // Every permutation of: + // - offset in [0, 4] + // - align in [1, 16] + // - elem_size in [0, 4] (plus no elem_size) + validate_against_rust!(0, 1); + validate_against_rust!(0, 1, 0); + validate_against_rust!(0, 1, 1); + validate_against_rust!(0, 1, 2); + validate_against_rust!(0, 1, 3); + validate_against_rust!(0, 1, 4); + validate_against_rust!(0, 2); + validate_against_rust!(0, 2, 0); + validate_against_rust!(0, 2, 1); + validate_against_rust!(0, 2, 2); + validate_against_rust!(0, 2, 3); + validate_against_rust!(0, 2, 4); + validate_against_rust!(0, 4); + validate_against_rust!(0, 4, 0); + validate_against_rust!(0, 4, 1); + validate_against_rust!(0, 4, 2); + validate_against_rust!(0, 4, 3); + validate_against_rust!(0, 4, 4); + validate_against_rust!(0, 8); + validate_against_rust!(0, 8, 0); + validate_against_rust!(0, 8, 1); + validate_against_rust!(0, 8, 2); + validate_against_rust!(0, 8, 3); + validate_against_rust!(0, 8, 4); + validate_against_rust!(0, 16); + validate_against_rust!(0, 16, 0); + validate_against_rust!(0, 16, 1); + validate_against_rust!(0, 16, 2); + validate_against_rust!(0, 16, 3); + validate_against_rust!(0, 16, 4); + validate_against_rust!(1, 1); + validate_against_rust!(1, 1, 0); + validate_against_rust!(1, 1, 1); + validate_against_rust!(1, 1, 2); + validate_against_rust!(1, 1, 3); + validate_against_rust!(1, 1, 4); + validate_against_rust!(1, 2); + validate_against_rust!(1, 2, 0); + validate_against_rust!(1, 2, 1); + validate_against_rust!(1, 2, 2); + validate_against_rust!(1, 2, 3); + validate_against_rust!(1, 2, 4); + validate_against_rust!(1, 4); + validate_against_rust!(1, 4, 0); + validate_against_rust!(1, 4, 1); + validate_against_rust!(1, 4, 2); + validate_against_rust!(1, 4, 3); + validate_against_rust!(1, 4, 4); + validate_against_rust!(1, 8); + validate_against_rust!(1, 8, 0); + validate_against_rust!(1, 8, 1); + validate_against_rust!(1, 8, 2); + validate_against_rust!(1, 8, 3); + validate_against_rust!(1, 8, 4); + validate_against_rust!(1, 16); + validate_against_rust!(1, 16, 0); + validate_against_rust!(1, 16, 1); + validate_against_rust!(1, 16, 2); + validate_against_rust!(1, 16, 3); + validate_against_rust!(1, 16, 4); + validate_against_rust!(2, 1); + validate_against_rust!(2, 1, 0); + validate_against_rust!(2, 1, 1); + validate_against_rust!(2, 1, 2); + validate_against_rust!(2, 1, 3); + validate_against_rust!(2, 1, 4); + validate_against_rust!(2, 2); + validate_against_rust!(2, 2, 0); + validate_against_rust!(2, 2, 1); + validate_against_rust!(2, 2, 2); + validate_against_rust!(2, 2, 3); + validate_against_rust!(2, 2, 4); + validate_against_rust!(2, 4); + validate_against_rust!(2, 4, 0); + validate_against_rust!(2, 4, 1); + validate_against_rust!(2, 4, 2); + validate_against_rust!(2, 4, 3); + validate_against_rust!(2, 4, 4); + validate_against_rust!(2, 8); + validate_against_rust!(2, 8, 0); + validate_against_rust!(2, 8, 1); + validate_against_rust!(2, 8, 2); + validate_against_rust!(2, 8, 3); + validate_against_rust!(2, 8, 4); + validate_against_rust!(2, 16); + validate_against_rust!(2, 16, 0); + validate_against_rust!(2, 16, 1); + validate_against_rust!(2, 16, 2); + validate_against_rust!(2, 16, 3); + validate_against_rust!(2, 16, 4); + validate_against_rust!(3, 1); + validate_against_rust!(3, 1, 0); + validate_against_rust!(3, 1, 1); + validate_against_rust!(3, 1, 2); + validate_against_rust!(3, 1, 3); + validate_against_rust!(3, 1, 4); + validate_against_rust!(3, 2); + validate_against_rust!(3, 2, 0); + validate_against_rust!(3, 2, 1); + validate_against_rust!(3, 2, 2); + validate_against_rust!(3, 2, 3); + validate_against_rust!(3, 2, 4); + validate_against_rust!(3, 4); + validate_against_rust!(3, 4, 0); + validate_against_rust!(3, 4, 1); + validate_against_rust!(3, 4, 2); + validate_against_rust!(3, 4, 3); + validate_against_rust!(3, 4, 4); + validate_against_rust!(3, 8); + validate_against_rust!(3, 8, 0); + validate_against_rust!(3, 8, 1); + validate_against_rust!(3, 8, 2); + validate_against_rust!(3, 8, 3); + validate_against_rust!(3, 8, 4); + validate_against_rust!(3, 16); + validate_against_rust!(3, 16, 0); + validate_against_rust!(3, 16, 1); + validate_against_rust!(3, 16, 2); + validate_against_rust!(3, 16, 3); + validate_against_rust!(3, 16, 4); + validate_against_rust!(4, 1); + validate_against_rust!(4, 1, 0); + validate_against_rust!(4, 1, 1); + validate_against_rust!(4, 1, 2); + validate_against_rust!(4, 1, 3); + validate_against_rust!(4, 1, 4); + validate_against_rust!(4, 2); + validate_against_rust!(4, 2, 0); + validate_against_rust!(4, 2, 1); + validate_against_rust!(4, 2, 2); + validate_against_rust!(4, 2, 3); + validate_against_rust!(4, 2, 4); + validate_against_rust!(4, 4); + validate_against_rust!(4, 4, 0); + validate_against_rust!(4, 4, 1); + validate_against_rust!(4, 4, 2); + validate_against_rust!(4, 4, 3); + validate_against_rust!(4, 4, 4); + validate_against_rust!(4, 8); + validate_against_rust!(4, 8, 0); + validate_against_rust!(4, 8, 1); + validate_against_rust!(4, 8, 2); + validate_against_rust!(4, 8, 3); + validate_against_rust!(4, 8, 4); + validate_against_rust!(4, 16); + validate_against_rust!(4, 16, 0); + validate_against_rust!(4, 16, 1); + validate_against_rust!(4, 16, 2); + validate_against_rust!(4, 16, 3); + validate_against_rust!(4, 16, 4); + } +} + +#[cfg(kani)] +mod proofs { + use core::alloc::Layout; + + use super::*; + + impl kani::Arbitrary for DstLayout { + fn any() -> Self { + let align: NonZeroUsize = kani::any(); + let size_info: SizeInfo = kani::any(); + + kani::assume(align.is_power_of_two()); + kani::assume(align < DstLayout::THEORETICAL_MAX_ALIGN); + + // For testing purposes, we most care about instantiations of + // `DstLayout` that can correspond to actual Rust types. We use + // `Layout` to verify that our `DstLayout` satisfies the validity + // conditions of Rust layouts. + kani::assume( + match size_info { + SizeInfo::Sized { size } => Layout::from_size_align(size, align.get()), + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size: _ }) => { + // `SliceDst` cannot encode an exact size, but we know + // it is at least `offset` bytes. + Layout::from_size_align(offset, align.get()) + } + } + .is_ok(), + ); + + Self { align: align, size_info: size_info, statically_shallow_unpadded: kani::any() } + } + } + + impl kani::Arbitrary for SizeInfo { + fn any() -> Self { + let is_sized: bool = kani::any(); + + match is_sized { + true => { + let size: usize = kani::any(); + + kani::assume(size <= DstLayout::MAX_SIZE); + + SizeInfo::Sized { size } + } + false => SizeInfo::SliceDst(kani::any()), + } + } + } + + impl kani::Arbitrary for TrailingSliceLayout { + fn any() -> Self { + let elem_size: usize = kani::any(); + let offset: usize = kani::any(); + + kani::assume(elem_size < DstLayout::MAX_SIZE); + kani::assume(offset < DstLayout::MAX_SIZE); + + TrailingSliceLayout { elem_size, offset } + } + } + + #[kani::proof] + fn prove_requires_dynamic_padding() { + let layout: DstLayout = kani::any(); + + let SizeInfo::SliceDst(size_info) = layout.size_info else { + kani::assume(false); + loop {} + }; + + let meta: usize = kani::any(); + + let Some(trailing_slice_size) = size_info.elem_size.checked_mul(meta) else { + // The `trailing_slice_size` exceeds `usize::MAX`; `meta` is invalid. + kani::assume(false); + loop {} + }; + + let Some(unpadded_size) = size_info.offset.checked_add(trailing_slice_size) else { + // The `unpadded_size` exceeds `usize::MAX`; `meta`` is invalid. + kani::assume(false); + loop {} + }; + + if unpadded_size >= DstLayout::MAX_SIZE { + // The `unpadded_size` exceeds `isize::MAX`; `meta` is invalid. + kani::assume(false); + loop {} + } + + let trailing_padding = util::padding_needed_for(unpadded_size, layout.align); + + if !layout.requires_dynamic_padding() { + assert!(trailing_padding == 0); + } + } + + #[kani::proof] + fn prove_dst_layout_extend() { + use crate::util::{max, min, padding_needed_for}; + + let base: DstLayout = kani::any(); + let field: DstLayout = kani::any(); + let packed: Option = kani::any(); + + if let Some(max_align) = packed { + kani::assume(max_align.is_power_of_two()); + kani::assume(base.align <= max_align); + } + + // The base can only be extended if it's sized. + kani::assume(matches!(base.size_info, SizeInfo::Sized { .. })); + let base_size = if let SizeInfo::Sized { size } = base.size_info { + size + } else { + unreachable!(); + }; + + // Under the above conditions, `DstLayout::extend` will not panic. + let composite = base.extend(field, packed); + + // The field's alignment is clamped by `max_align` (i.e., the + // `packed` attribute, if any) [1]. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // The alignments of each field, for the purpose of positioning + // fields, is the smaller of the specified alignment and the + // alignment of the field's type. + let field_align = min(field.align, packed.unwrap_or(DstLayout::THEORETICAL_MAX_ALIGN)); + + // The struct's alignment is the maximum of its previous alignment and + // `field_align`. + assert_eq!(composite.align, max(base.align, field_align)); + + // Compute the minimum amount of inter-field padding needed to + // satisfy the field's alignment, and offset of the trailing field. + // [1] + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // Inter-field padding is guaranteed to be the minimum required in + // order to satisfy each field's (possibly altered) alignment. + let padding = padding_needed_for(base_size, field_align); + let offset = base_size + padding; + + // For testing purposes, we'll also construct `alloc::Layout` + // stand-ins for `DstLayout`, and show that `extend` behaves + // comparably on both types. + let base_analog = Layout::from_size_align(base_size, base.align.get()).unwrap(); + + match field.size_info { + SizeInfo::Sized { size: field_size } => { + if let SizeInfo::Sized { size: composite_size } = composite.size_info { + // If the trailing field is sized, the resulting layout will + // be sized. Its size will be the sum of the preceding + // layout, the size of the new field, and the size of + // inter-field padding between the two. + assert_eq!(composite_size, offset + field_size); + + let field_analog = + Layout::from_size_align(field_size, field_align.get()).unwrap(); + + if let Ok((actual_composite, actual_offset)) = base_analog.extend(field_analog) + { + assert_eq!(actual_offset, offset); + assert_eq!(actual_composite.size(), composite_size); + assert_eq!(actual_composite.align(), composite.align.get()); + } else { + // An error here reflects that composite of `base` + // and `field` cannot correspond to a real Rust type + // fragment, because such a fragment would violate + // the basic invariants of a valid Rust layout. At + // the time of writing, `DstLayout` is a little more + // permissive than `Layout`, so we don't assert + // anything in this branch (e.g., unreachability). + } + } else { + panic!("The composite of two sized layouts must be sized.") + } + } + SizeInfo::SliceDst(TrailingSliceLayout { + offset: field_offset, + elem_size: field_elem_size, + }) => { + if let SizeInfo::SliceDst(TrailingSliceLayout { + offset: composite_offset, + elem_size: composite_elem_size, + }) = composite.size_info + { + // The offset of the trailing slice component is the sum + // of the offset of the trailing field and the trailing + // slice offset within that field. + assert_eq!(composite_offset, offset + field_offset); + // The elem size is unchanged. + assert_eq!(composite_elem_size, field_elem_size); + + let field_analog = + Layout::from_size_align(field_offset, field_align.get()).unwrap(); + + if let Ok((actual_composite, actual_offset)) = base_analog.extend(field_analog) + { + assert_eq!(actual_offset, offset); + assert_eq!(actual_composite.size(), composite_offset); + assert_eq!(actual_composite.align(), composite.align.get()); + } else { + // An error here reflects that composite of `base` + // and `field` cannot correspond to a real Rust type + // fragment, because such a fragment would violate + // the basic invariants of a valid Rust layout. At + // the time of writing, `DstLayout` is a little more + // permissive than `Layout`, so we don't assert + // anything in this branch (e.g., unreachability). + } + } else { + panic!("The extension of a layout with a DST must result in a DST.") + } + } + } + } + + #[kani::proof] + #[kani::should_panic] + fn prove_dst_layout_extend_dst_panics() { + let base: DstLayout = kani::any(); + let field: DstLayout = kani::any(); + let packed: Option = kani::any(); + + if let Some(max_align) = packed { + kani::assume(max_align.is_power_of_two()); + kani::assume(base.align <= max_align); + } + + kani::assume(matches!(base.size_info, SizeInfo::SliceDst(..))); + + let _ = base.extend(field, packed); + } + + #[kani::proof] + fn prove_dst_layout_pad_to_align() { + use crate::util::padding_needed_for; + + let layout: DstLayout = kani::any(); + + let padded = layout.pad_to_align(); + + // Calling `pad_to_align` does not alter the `DstLayout`'s alignment. + assert_eq!(padded.align, layout.align); + + if let SizeInfo::Sized { size: unpadded_size } = layout.size_info { + if let SizeInfo::Sized { size: padded_size } = padded.size_info { + // If the layout is sized, it will remain sized after padding is + // added. Its sum will be its unpadded size and the size of the + // trailing padding needed to satisfy its alignment + // requirements. + let padding = padding_needed_for(unpadded_size, layout.align); + assert_eq!(padded_size, unpadded_size + padding); + + // Prove that calling `DstLayout::pad_to_align` behaves + // identically to `Layout::pad_to_align`. + let layout_analog = + Layout::from_size_align(unpadded_size, layout.align.get()).unwrap(); + let padded_analog = layout_analog.pad_to_align(); + assert_eq!(padded_analog.align(), layout.align.get()); + assert_eq!(padded_analog.size(), padded_size); + } else { + panic!("The padding of a sized layout must result in a sized layout.") + } + } else { + // If the layout is a DST, padding cannot be statically added. + assert_eq!(padded.size_info, layout.size_info); + } + } +} diff --git a/rust/zerocopy/src/lib.rs b/rust/zerocopy/src/lib.rs new file mode 100644 index 000000000000..89a696f732b8 --- /dev/null +++ b/rust/zerocopy/src/lib.rs @@ -0,0 +1,7610 @@ +// Copyright 2018 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +// After updating the following doc comment, make sure to run the following +// command to update `README.md` based on its contents: +// +// cargo -q run --manifest-path tools/Cargo.toml -p generate-readme > README.md + +//! ***Fast, safe, compile error. Pick two.*** +//! +//! Zerocopy makes zero-cost memory manipulation effortless. We write `unsafe` +//! so you don't have to. +//! +//! *For an overview of what's changed from zerocopy 0.7, check out our [release +//! notes][release-notes], which include a step-by-step upgrading guide.* +//! +//! *Have questions? Need more out of zerocopy? Submit a [customer request +//! issue][customer-request-issue] or ask the maintainers on +//! [GitHub][github-q-a] or [Discord][discord]!* +//! +//! [customer-request-issue]: https://github.com/google/zerocopy/issues/new/choose +//! [release-notes]: https://github.com/google/zerocopy/discussions/1680 +//! [github-q-a]: https://github.com/google/zerocopy/discussions/categories/q-a +//! [discord]: https://discord.gg/MAvWH2R6zk +//! +//! # Overview +//! +//! ##### Conversion Traits +//! +//! Zerocopy provides four derivable traits for zero-cost conversions: +//! - [`TryFromBytes`] indicates that a type may safely be converted from +//! certain byte sequences (conditional on runtime checks) +//! - [`FromZeros`] indicates that a sequence of zero bytes represents a valid +//! instance of a type +//! - [`FromBytes`] indicates that a type may safely be converted from an +//! arbitrary byte sequence +//! - [`IntoBytes`] indicates that a type may safely be converted *to* a byte +//! sequence +//! +//! These traits support sized types, slices, and [slice DSTs][slice-dsts]. +//! +//! [slice-dsts]: KnownLayout#dynamically-sized-types +//! +//! ##### Marker Traits +//! +//! Zerocopy provides three derivable marker traits that do not provide any +//! functionality themselves, but are required to call certain methods provided +//! by the conversion traits: +//! - [`KnownLayout`] indicates that zerocopy can reason about certain layout +//! qualities of a type +//! - [`Immutable`] indicates that a type is free from interior mutability, +//! except by ownership or an exclusive (`&mut`) borrow +//! - [`Unaligned`] indicates that a type's alignment requirement is 1 +//! +//! You should generally derive these marker traits whenever possible. +//! +//! ##### Conversion Macros +//! +//! Zerocopy provides six macros for safe casting between types: +//! +//! - ([`try_`][try_transmute])[`transmute`] (conditionally) converts a value of +//! one type to a value of another type of the same size +//! - ([`try_`][try_transmute_mut])[`transmute_mut`] (conditionally) converts a +//! mutable reference of one type to a mutable reference of another type of +//! the same size +//! - ([`try_`][try_transmute_ref])[`transmute_ref`] (conditionally) converts a +//! mutable or immutable reference of one type to an immutable reference of +//! another type of the same size +//! +//! These macros perform *compile-time* size and alignment checks, meaning that +//! unconditional casts have zero cost at runtime. Conditional casts do not need +//! to validate size or alignment runtime, but do need to validate contents. +//! +//! These macros cannot be used in generic contexts. For generic conversions, +//! use the methods defined by the [conversion traits](#conversion-traits). +//! +//! ##### Byteorder-Aware Numerics +//! +//! Zerocopy provides byte-order aware integer types that support these +//! conversions; see the [`byteorder`] module. These types are especially useful +//! for network parsing. +//! +//! # Cargo Features +//! +//! - **`alloc`** +//! By default, `zerocopy` is `no_std`. When the `alloc` feature is enabled, +//! the `alloc` crate is added as a dependency, and some allocation-related +//! functionality is added. +//! +//! - **`std`** +//! By default, `zerocopy` is `no_std`. When the `std` feature is enabled, the +//! `std` crate is added as a dependency (ie, `no_std` is disabled), and +//! support for some `std` types is added. `std` implies `alloc`. +//! +//! - **`derive`** +//! Provides derives for the core marker traits via the `zerocopy-derive` +//! crate. These derives are re-exported from `zerocopy`, so it is not +//! necessary to depend on `zerocopy-derive` directly. +//! +//! However, you may experience better compile times if you instead directly +//! depend on both `zerocopy` and `zerocopy-derive` in your `Cargo.toml`, +//! since doing so will allow Rust to compile these crates in parallel. To do +//! so, do *not* enable the `derive` feature, and list both dependencies in +//! your `Cargo.toml` with the same leading non-zero version number; e.g: +//! +//! ```toml +//! [dependencies] +//! zerocopy = "0.X" +//! zerocopy-derive = "0.X" +//! ``` +//! +//! To avoid the risk of [duplicate import errors][duplicate-import-errors] if +//! one of your dependencies enables zerocopy's `derive` feature, import +//! derives as `use zerocopy_derive::*` rather than by name (e.g., `use +//! zerocopy_derive::FromBytes`). +//! +//! - **`simd`** +//! When the `simd` feature is enabled, `FromZeros`, `FromBytes`, and +//! `IntoBytes` impls are emitted for all stable SIMD types which exist on the +//! target platform. Note that the layout of SIMD types is not yet stabilized, +//! so these impls may be removed in the future if layout changes make them +//! invalid. For more information, see the Unsafe Code Guidelines Reference +//! page on the [layout of packed SIMD vectors][simd-layout]. +//! +//! - **`simd-nightly`** +//! Enables the `simd` feature and adds support for SIMD types which are only +//! available on nightly. Since these types are unstable, support for any type +//! may be removed at any point in the future. +//! +//! - **`float-nightly`** +//! Adds support for the unstable `f16` and `f128` types. These types are +//! not yet fully implemented and may not be supported on all platforms. +//! +//! [duplicate-import-errors]: https://github.com/google/zerocopy/issues/1587 +//! [simd-layout]: https://rust-lang.github.io/unsafe-code-guidelines/layout/packed-simd-vectors.html +//! +//! # Build Tuning +//! +//! ## `--cfg zerocopy_inline_always` +//! +//! Upgrades `#[inline]` to `#[inline(always)]` on many of zerocopy's public +//! functions and methods. This provides a narrowly-scoped alternative that +//! *may* improve the optimization of hot paths using zerocopy without the broad +//! compile-time penalties of configuring `codegen-units=1`. +//! +//! # Security Ethos +//! +//! Zerocopy is expressly designed for use in security-critical contexts. We +//! strive to ensure that that zerocopy code is sound under Rust's current +//! memory model, and *any future memory model*. We ensure this by: +//! - **...not 'guessing' about Rust's semantics.** +//! We annotate `unsafe` code with a precise rationale for its soundness that +//! cites a relevant section of Rust's official documentation. When Rust's +//! documented semantics are unclear, we work with the Rust Operational +//! Semantics Team to clarify Rust's documentation. +//! - **...rigorously testing our implementation.** +//! We run tests using [Miri], ensuring that zerocopy is sound across a wide +//! array of supported target platforms of varying endianness and pointer +//! width, and across both current and experimental memory models of Rust. +//! - **...formally proving the correctness of our implementation.** +//! We apply formal verification tools like [Kani][kani] to prove zerocopy's +//! correctness. +//! +//! For more information, see our full [soundness policy]. +//! +//! [Miri]: https://github.com/rust-lang/miri +//! [Kani]: https://github.com/model-checking/kani +//! [soundness policy]: https://github.com/google/zerocopy/blob/main/POLICIES.md#soundness +//! +//! # Relationship to Project Safe Transmute +//! +//! [Project Safe Transmute] is an official initiative of the Rust Project to +//! develop language-level support for safer transmutation. The Project consults +//! with crates like zerocopy to identify aspects of safer transmutation that +//! would benefit from compiler support, and has developed an [experimental, +//! compiler-supported analysis][mcp-transmutability] which determines whether, +//! for a given type, any value of that type may be soundly transmuted into +//! another type. Once this functionality is sufficiently mature, zerocopy +//! intends to replace its internal transmutability analysis (implemented by our +//! custom derives) with the compiler-supported one. This change will likely be +//! an implementation detail that is invisible to zerocopy's users. +//! +//! Project Safe Transmute will not replace the need for most of zerocopy's +//! higher-level abstractions. The experimental compiler analysis is a tool for +//! checking the soundness of `unsafe` code, not a tool to avoid writing +//! `unsafe` code altogether. For the foreseeable future, crates like zerocopy +//! will still be required in order to provide higher-level abstractions on top +//! of the building block provided by Project Safe Transmute. +//! +//! [Project Safe Transmute]: https://rust-lang.github.io/rfcs/2835-project-safe-transmute.html +//! [mcp-transmutability]: https://github.com/rust-lang/compiler-team/issues/411 +//! +//! # MSRV +//! +//! See our [MSRV policy]. +//! +//! [MSRV policy]: https://github.com/google/zerocopy/blob/main/POLICIES.md#msrv +//! +//! # Changelog +//! +//! Zerocopy uses [GitHub Releases]. +//! +//! [GitHub Releases]: https://github.com/google/zerocopy/releases +//! +//! # Thanks +//! +//! Zerocopy is maintained by engineers at Google with help from [many wonderful +//! contributors][contributors]. Thank you to everyone who has lent a hand in +//! making Rust a little more secure! +//! +//! [contributors]: https://github.com/google/zerocopy/graphs/contributors + +// Sometimes we want to use lints which were added after our MSRV. +// `unknown_lints` is `warn` by default and we deny warnings in CI, so without +// this attribute, any unknown lint would cause a CI failure when testing with +// our MSRV. +#![allow(unknown_lints, non_local_definitions, unreachable_patterns)] +#![deny(renamed_and_removed_lints)] +#![deny( + anonymous_parameters, + deprecated_in_future, + late_bound_lifetime_arguments, + missing_copy_implementations, + missing_debug_implementations, + missing_docs, + path_statements, + patterns_in_fns_without_body, + rust_2018_idioms, + trivial_numeric_casts, + unreachable_pub, + unsafe_op_in_unsafe_fn, + unused_extern_crates, + // We intentionally choose not to deny `unused_qualifications`. When items + // are added to the prelude (e.g., `core::mem::size_of`), this has the + // consequence of making some uses trigger this lint on the latest toolchain + // (e.g., `mem::size_of`), but fixing it (e.g. by replacing with `size_of`) + // does not work on older toolchains. + // + // We tested a more complicated fix in #1413, but ultimately decided that, + // since this lint is just a minor style lint, the complexity isn't worth it + // - it's fine to occasionally have unused qualifications slip through, + // especially since these do not affect our user-facing API in any way. + variant_size_differences +)] +#![cfg_attr( + __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, + deny(fuzzy_provenance_casts, lossy_provenance_casts) +)] +#![deny( + clippy::all, + clippy::alloc_instead_of_core, + clippy::arithmetic_side_effects, + clippy::as_underscore, + clippy::assertions_on_result_states, + clippy::as_conversions, + clippy::correctness, + clippy::dbg_macro, + clippy::decimal_literal_representation, + clippy::double_must_use, + clippy::get_unwrap, + clippy::indexing_slicing, + clippy::missing_inline_in_public_items, + clippy::missing_safety_doc, + clippy::multiple_unsafe_ops_per_block, + clippy::must_use_candidate, + clippy::must_use_unit, + clippy::obfuscated_if_else, + clippy::perf, + clippy::print_stdout, + clippy::return_self_not_must_use, + clippy::std_instead_of_core, + clippy::style, + clippy::suspicious, + clippy::todo, + clippy::undocumented_unsafe_blocks, + clippy::unimplemented, + clippy::unnested_or_patterns, + clippy::unwrap_used, + clippy::use_debug +)] +// `clippy::incompatible_msrv` (implied by `clippy::suspicious`): This sometimes +// has false positives, and we test on our MSRV in CI, so it doesn't help us +// anyway. +#![allow(clippy::needless_lifetimes, clippy::type_complexity, clippy::incompatible_msrv)] +#![deny( + rustdoc::bare_urls, + rustdoc::broken_intra_doc_links, + rustdoc::invalid_codeblock_attributes, + rustdoc::invalid_html_tags, + rustdoc::invalid_rust_codeblocks, + rustdoc::missing_crate_level_docs, + rustdoc::private_intra_doc_links +)] +// In test code, it makes sense to weight more heavily towards concise, readable +// code over correct or debuggable code. +#![cfg_attr(any(test, kani), allow( + // In tests, you get line numbers and have access to source code, so panic + // messages are less important. You also often unwrap a lot, which would + // make expect'ing instead very verbose. + clippy::unwrap_used, + // In tests, there's no harm to "panic risks" - the worst that can happen is + // that your test will fail, and you'll fix it. By contrast, panic risks in + // production code introduce the possibly of code panicking unexpectedly "in + // the field". + clippy::arithmetic_side_effects, + clippy::indexing_slicing, +))] +#![cfg_attr(not(any(test, kani, feature = "std")), no_std)] +#![cfg_attr( + all(feature = "simd-nightly", target_arch = "arm"), + feature(stdarch_arm_neon_intrinsics) +)] +#![cfg_attr( + all(feature = "simd-nightly", any(target_arch = "powerpc", target_arch = "powerpc64")), + feature(stdarch_powerpc) +)] +#![cfg_attr(feature = "float-nightly", feature(f16, f128))] +#![cfg_attr(doc_cfg, feature(doc_cfg))] +#![cfg_attr(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, feature(coverage_attribute))] +#![cfg_attr( + any(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, miri), + feature(layout_for_ptr) +)] +#![cfg_attr(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), feature(test))] + +// This is a hack to allow zerocopy-derive derives to work in this crate. They +// assume that zerocopy is linked as an extern crate, so they access items from +// it as `zerocopy::Xxx`. This makes that still work. +#[cfg(any(feature = "derive", test))] +extern crate self as zerocopy; + +#[cfg(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS))] +extern crate test; + +#[doc(hidden)] +#[macro_use] +pub mod util; + +pub mod byte_slice; +pub mod byteorder; +mod deprecated; + +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE)] +pub mod doctests; + +// This module is `pub` so that zerocopy's error types and error handling +// documentation is grouped together in a cohesive module. In practice, we +// expect most users to use the re-export of `error`'s items to avoid identifier +// stuttering. +pub mod error; +mod impls; +#[doc(hidden)] +pub mod layout; +mod macros; +#[cfg_attr(not(zerocopy_unstable_ptr), doc(hidden))] +#[cfg_attr(doc_cfg, doc(cfg(zerocopy_unstable_ptr)))] +pub mod pointer; +mod r#ref; +mod split_at; +// FIXME(#252): If we make this pub, come up with a better name. +mod wrappers; + +use core::{ + cell::{Cell, UnsafeCell}, + cmp::Ordering, + fmt::{self, Debug, Display, Formatter}, + hash::Hasher, + marker::PhantomData, + mem::{self, ManuallyDrop, MaybeUninit as CoreMaybeUninit}, + num::{ + NonZeroI128, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroIsize, NonZeroU128, + NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize, Wrapping, + }, + ops::{Deref, DerefMut}, + ptr::{self, NonNull}, + slice, +}; +#[cfg(feature = "std")] +use std::io; + +#[doc(hidden)] +pub use crate::pointer::{ + invariant::{self, BecauseExclusive}, + PtrInner, +}; +pub use crate::{ + byte_slice::*, + byteorder::*, + error::*, + r#ref::*, + split_at::{Split, SplitAt}, + wrappers::*, +}; + +#[cfg(any(feature = "alloc", test, kani))] +extern crate alloc; +#[cfg(any(feature = "alloc", test))] +use alloc::{boxed::Box, vec::Vec}; +#[cfg(any(feature = "alloc", test))] +use core::alloc::Layout; + +// Used by `KnownLayout`. +#[doc(hidden)] +pub use crate::layout::*; +// Used by `TryFromBytes::is_bit_valid`. +#[doc(hidden)] +pub use crate::pointer::{invariant::BecauseImmutable, Maybe, Ptr}; +// For each trait polyfill, as soon as the corresponding feature is stable, the +// polyfill import will be unused because method/function resolution will prefer +// the inherent method/function over a trait method/function. Thus, we suppress +// the `unused_imports` warning. +// +// See the documentation on `util::polyfills` for more information. +#[allow(unused_imports)] +use crate::util::polyfills::{self, NonNullExt as _, NumExt as _}; +#[cfg_attr(not(zerocopy_unstable_ptr), doc(hidden))] +#[cfg_attr(doc_cfg, doc(cfg(zerocopy_unstable_ptr)))] +pub use crate::util::MetadataOf; + +#[cfg(all(test, not(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE)))] +const _: () = { + #[deprecated = "Development of zerocopy using cargo is not supported. Please use `cargo.sh` or `win-cargo.bat` instead."] + #[allow(unused)] + const WARNING: () = (); + #[warn(deprecated)] + WARNING +}; + +/// Implements [`KnownLayout`]. +/// +/// This derive analyzes various aspects of a type's layout that are needed for +/// some of zerocopy's APIs. It can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::KnownLayout; +/// #[derive(KnownLayout)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Limitations +/// +/// This derive cannot currently be applied to unsized structs without an +/// explicit `repr` attribute. +/// +/// Some invocations of this derive run afoul of a [known bug] in Rust's type +/// privacy checker. For example, this code: +/// +/// ```compile_fail,E0446 +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(KnownLayout)] +/// #[repr(C)] +/// pub struct PublicType { +/// leading: Foo, +/// trailing: Bar, +/// } +/// +/// #[derive(KnownLayout)] +/// struct Foo; +/// +/// #[derive(KnownLayout)] +/// struct Bar; +/// ``` +/// +/// ...results in a compilation error: +/// +/// ```text +/// error[E0446]: private type `Bar` in public interface +/// --> examples/bug.rs:3:10 +/// | +/// 3 | #[derive(KnownLayout)] +/// | ^^^^^^^^^^^ can't leak private type +/// ... +/// 14 | struct Bar; +/// | ---------- `Bar` declared as private +/// | +/// = note: this error originates in the derive macro `KnownLayout` (in Nightly builds, run with -Z macro-backtrace for more info) +/// ``` +/// +/// This issue arises when `#[derive(KnownLayout)]` is applied to `repr(C)` +/// structs whose trailing field type is less public than the enclosing struct. +/// +/// To work around this, mark the trailing field type `pub` and annotate it with +/// `#[doc(hidden)]`; e.g.: +/// +/// ```no_run +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(KnownLayout)] +/// #[repr(C)] +/// pub struct PublicType { +/// leading: Foo, +/// trailing: Bar, +/// } +/// +/// #[derive(KnownLayout)] +/// struct Foo; +/// +/// #[doc(hidden)] +/// #[derive(KnownLayout)] +/// pub struct Bar; // <- `Bar` is now also `pub` +/// ``` +/// +/// [known bug]: https://github.com/rust-lang/rust/issues/45713 +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::KnownLayout; +// These exist so that code which was written against the old names will get +// less confusing error messages when they upgrade to a more recent version of +// zerocopy. On our MSRV toolchain, the error messages read, for example: +// +// error[E0603]: trait `FromZeroes` is private +// --> examples/deprecated.rs:1:15 +// | +// 1 | use zerocopy::FromZeroes; +// | ^^^^^^^^^^ private trait +// | +// note: the trait `FromZeroes` is defined here +// --> /Users/josh/workspace/zerocopy/src/lib.rs:1845:5 +// | +// 1845 | use FromZeros as FromZeroes; +// | ^^^^^^^^^^^^^^^^^^^^^^^ +// +// The "note" provides enough context to make it easy to figure out how to fix +// the error. +#[allow(unused)] +use {FromZeros as FromZeroes, IntoBytes as AsBytes, Ref as LayoutVerified}; + +/// Indicates that zerocopy can reason about certain aspects of a type's layout. +/// +/// This trait is required by many of zerocopy's APIs. It supports sized types, +/// slices, and [slice DSTs](#dynamically-sized-types). +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(KnownLayout)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::KnownLayout; +/// #[derive(KnownLayout)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// enum MyEnum { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated analysis to deduce the layout +/// characteristics of types. You **must** implement this trait via the derive. +/// +/// # Dynamically-sized types +/// +/// `KnownLayout` supports slice-based dynamically sized types ("slice DSTs"). +/// +/// A slice DST is a type whose trailing field is either a slice or another +/// slice DST, rather than a type with fixed size. For example: +/// +/// ``` +/// #[repr(C)] +/// struct PacketHeader { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[repr(C)] +/// struct Packet { +/// header: PacketHeader, +/// body: [u8], +/// } +/// ``` +/// +/// It can be useful to think of slice DSTs as a generalization of slices - in +/// other words, a normal slice is just the special case of a slice DST with +/// zero leading fields. In particular: +/// - Like slices, slice DSTs can have different lengths at runtime +/// - Like slices, slice DSTs cannot be passed by-value, but only by reference +/// or via other indirection such as `Box` +/// - Like slices, a reference (or `Box`, or other pointer type) to a slice DST +/// encodes the number of elements in the trailing slice field +/// +/// ## Slice DST layout +/// +/// Just like other composite Rust types, the layout of a slice DST is not +/// well-defined unless it is specified using an explicit `#[repr(...)]` +/// attribute such as `#[repr(C)]`. [Other representations are +/// supported][reprs], but in this section, we'll use `#[repr(C)]` as our +/// example. +/// +/// A `#[repr(C)]` slice DST is laid out [just like sized `#[repr(C)]` +/// types][repr-c-structs], but the presence of a variable-length field +/// introduces the possibility of *dynamic padding*. In particular, it may be +/// necessary to add trailing padding *after* the trailing slice field in order +/// to satisfy the outer type's alignment, and the amount of padding required +/// may be a function of the length of the trailing slice field. This is just a +/// natural consequence of the normal `#[repr(C)]` rules applied to slice DSTs, +/// but it can result in surprising behavior. For example, consider the +/// following type: +/// +/// ``` +/// #[repr(C)] +/// struct Foo { +/// a: u32, +/// b: u8, +/// z: [u16], +/// } +/// ``` +/// +/// Assuming that `u32` has alignment 4 (this is not true on all platforms), +/// then `Foo` has alignment 4 as well. Here is the smallest possible value for +/// `Foo`: +/// +/// ```text +/// byte offset | 01234567 +/// field | aaaab--- +/// >< +/// ``` +/// +/// In this value, `z` has length 0. Abiding by `#[repr(C)]`, the lowest offset +/// that we can place `z` at is 5, but since `z` has alignment 2, we need to +/// round up to offset 6. This means that there is one byte of padding between +/// `b` and `z`, then 0 bytes of `z` itself (denoted `><` in this diagram), and +/// then two bytes of padding after `z` in order to satisfy the overall +/// alignment of `Foo`. The size of this instance is 8 bytes. +/// +/// What about if `z` has length 1? +/// +/// ```text +/// byte offset | 01234567 +/// field | aaaab-zz +/// ``` +/// +/// In this instance, `z` has length 1, and thus takes up 2 bytes. That means +/// that we no longer need padding after `z` in order to satisfy `Foo`'s +/// alignment. We've now seen two different values of `Foo` with two different +/// lengths of `z`, but they both have the same size - 8 bytes. +/// +/// What about if `z` has length 2? +/// +/// ```text +/// byte offset | 012345678901 +/// field | aaaab-zzzz-- +/// ``` +/// +/// Now `z` has length 2, and thus takes up 4 bytes. This brings our un-padded +/// size to 10, and so we now need another 2 bytes of padding after `z` to +/// satisfy `Foo`'s alignment. +/// +/// Again, all of this is just a logical consequence of the `#[repr(C)]` rules +/// applied to slice DSTs, but it can be surprising that the amount of trailing +/// padding becomes a function of the trailing slice field's length, and thus +/// can only be computed at runtime. +/// +/// [reprs]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// [repr-c-structs]: https://doc.rust-lang.org/reference/type-layout.html#reprc-structs +/// +/// ## What is a valid size? +/// +/// There are two places in zerocopy's API that we refer to "a valid size" of a +/// type. In normal casts or conversions, where the source is a byte slice, we +/// need to know whether the source byte slice is a valid size of the +/// destination type. In prefix or suffix casts, we need to know whether *there +/// exists* a valid size of the destination type which fits in the source byte +/// slice and, if so, what the largest such size is. +/// +/// As outlined above, a slice DST's size is defined by the number of elements +/// in its trailing slice field. However, there is not necessarily a 1-to-1 +/// mapping between trailing slice field length and overall size. As we saw in +/// the previous section with the type `Foo`, instances with both 0 and 1 +/// elements in the trailing `z` field result in a `Foo` whose size is 8 bytes. +/// +/// When we say "x is a valid size of `T`", we mean one of two things: +/// - If `T: Sized`, then we mean that `x == size_of::()` +/// - If `T` is a slice DST, then we mean that there exists a `len` such that the instance of +/// `T` with `len` trailing slice elements has size `x` +/// +/// When we say "largest possible size of `T` that fits in a byte slice", we +/// mean one of two things: +/// - If `T: Sized`, then we mean `size_of::()` if the byte slice is at least +/// `size_of::()` bytes long +/// - If `T` is a slice DST, then we mean to consider all values, `len`, such +/// that the instance of `T` with `len` trailing slice elements fits in the +/// byte slice, and to choose the largest such `len`, if any +/// +/// +/// # Safety +/// +/// This trait does not convey any safety guarantees to code outside this crate. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `KnownLayout`. Future +/// releases of zerocopy may make backwards-breaking changes to these items, +/// including changes that only affect soundness, which may cause code which +/// uses those items to silently become unsound. +/// +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::KnownLayout")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.KnownLayout.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(KnownLayout)]` to `{Self}`") +)] +pub unsafe trait KnownLayout { + // The `Self: Sized` bound makes it so that `KnownLayout` can still be + // object safe. It's not currently object safe thanks to `const LAYOUT`, and + // it likely won't be in the future, but there's no reason not to be + // forwards-compatible with object safety. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type of metadata stored in a pointer to `Self`. + /// + /// This is `()` for sized types and [`usize`] for slice DSTs. + type PointerMetadata: PointerMetadata; + + /// A maybe-uninitialized analog of `Self` + /// + /// # Safety + /// + /// `Self::LAYOUT` and `Self::MaybeUninit::LAYOUT` are identical. + /// `Self::MaybeUninit` admits uninitialized bytes in all positions. + #[doc(hidden)] + type MaybeUninit: ?Sized + KnownLayout; + + /// The layout of `Self`. + /// + /// # Safety + /// + /// Callers may assume that `LAYOUT` accurately reflects the layout of + /// `Self`. In particular: + /// - `LAYOUT.align` is equal to `Self`'s alignment + /// - If `Self: Sized`, then `LAYOUT.size_info == SizeInfo::Sized { size }` + /// where `size == size_of::()` + /// - If `Self` is a slice DST, then `LAYOUT.size_info == + /// SizeInfo::SliceDst(slice_layout)` where: + /// - The size, `size`, of an instance of `Self` with `elems` trailing + /// slice elements is equal to `slice_layout.offset + + /// slice_layout.elem_size * elems` rounded up to the nearest multiple + /// of `LAYOUT.align` + /// - For such an instance, any bytes in the range `[slice_layout.offset + + /// slice_layout.elem_size * elems, size)` are padding and must not be + /// assumed to be initialized + #[doc(hidden)] + const LAYOUT: DstLayout; + + /// SAFETY: The returned pointer has the same address and provenance as + /// `bytes`. If `Self` is a DST, the returned pointer's referent has `elems` + /// elements in its trailing slice. + #[doc(hidden)] + fn raw_from_ptr_len(bytes: NonNull, meta: Self::PointerMetadata) -> NonNull; + + /// Extracts the metadata from a pointer to `Self`. + /// + /// # Safety + /// + /// `pointer_to_metadata` always returns the correct metadata stored in + /// `ptr`. + #[doc(hidden)] + fn pointer_to_metadata(ptr: *mut Self) -> Self::PointerMetadata; + + /// Computes the length of the byte range addressed by `ptr`. + /// + /// Returns `None` if the resulting length would not fit in an `usize`. + /// + /// # Safety + /// + /// Callers may assume that `size_of_val_raw` always returns the correct + /// size. + /// + /// Callers may assume that, if `ptr` addresses a byte range whose length + /// fits in an `usize`, this will return `Some`. + #[doc(hidden)] + #[must_use] + #[inline(always)] + fn size_of_val_raw(ptr: NonNull) -> Option { + let meta = Self::pointer_to_metadata(ptr.as_ptr()); + // SAFETY: `size_for_metadata` promises to only return `None` if the + // resulting size would not fit in a `usize`. + Self::size_for_metadata(meta) + } + + #[doc(hidden)] + #[must_use] + #[inline(always)] + fn raw_dangling() -> NonNull { + let meta = Self::PointerMetadata::from_elem_count(0); + Self::raw_from_ptr_len(NonNull::dangling(), meta) + } + + /// Computes the size of an object of type `Self` with the given pointer + /// metadata. + /// + /// # Safety + /// + /// `size_for_metadata` promises to return `None` if and only if the + /// resulting size would not fit in a [`usize`]. Note that the returned size + /// could exceed the actual maximum valid size of an allocated object, + /// [`isize::MAX`]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::KnownLayout; + /// + /// assert_eq!(u8::size_for_metadata(()), Some(1)); + /// assert_eq!(u16::size_for_metadata(()), Some(2)); + /// assert_eq!(<[u8]>::size_for_metadata(42), Some(42)); + /// assert_eq!(<[u16]>::size_for_metadata(42), Some(84)); + /// + /// // This size exceeds the maximum valid object size (`isize::MAX`): + /// assert_eq!(<[u8]>::size_for_metadata(usize::MAX), Some(usize::MAX)); + /// + /// // This size, if computed, would exceed `usize::MAX`: + /// assert_eq!(<[u16]>::size_for_metadata(usize::MAX), None); + /// ``` + #[inline(always)] + fn size_for_metadata(meta: Self::PointerMetadata) -> Option { + meta.size_for_metadata(Self::LAYOUT) + } + + /// Computes whether `meta` can describe a valid allocation of `Self`. + /// + /// # Safety + /// + /// `is_valid_metadata` promises to return `true` if and only if the size of + /// an allocation of `Self` with `meta` would not overflow an + /// [`isize::MAX`]. + #[doc(hidden)] + #[inline(always)] + fn is_valid_metadata(meta: Self::PointerMetadata) -> bool { + meta.to_elem_count() <= maximum_trailing_slice_len::().to_elem_count() + } +} + +/// Efficiently produces the [`TrailingSliceLayout`] of `T`. +#[inline(always)] +pub(crate) fn trailing_slice_layout() -> TrailingSliceLayout +where + T: ?Sized + KnownLayout, +{ + trait LayoutFacts { + const SIZE_INFO: TrailingSliceLayout; + } + + impl LayoutFacts for T + where + T: KnownLayout, + { + const SIZE_INFO: TrailingSliceLayout = match T::LAYOUT.size_info { + crate::SizeInfo::Sized { .. } => const_panic!("unreachable"), + crate::SizeInfo::SliceDst(info) => info, + }; + } + + T::SIZE_INFO +} + +/// Efficiently produces the maximum trailing slice length `T`. +#[inline(always)] +pub(crate) fn maximum_trailing_slice_len() -> usize +where + T: ?Sized + KnownLayout, +{ + trait LayoutFacts { + const MAX_LEN: usize; + } + + impl LayoutFacts for T + where + T: KnownLayout, + { + const MAX_LEN: usize = match T::LAYOUT.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { elem_size: 0, .. }) => usize::MAX, + _ => match T::LAYOUT.validate_cast_and_convert_metadata( + T::LAYOUT.align.get(), + DstLayout::MAX_SIZE, + CastType::Prefix, + ) { + Ok((elems, _)) => elems, + Err(_) => const_panic!("unreachable"), + }, + }; + } + + T::MAX_LEN +} + +/// The metadata associated with a [`KnownLayout`] type. +#[doc(hidden)] +pub trait PointerMetadata: Copy + Eq + Debug + Ord { + /// Constructs a `Self` from an element count. + /// + /// If `Self = ()`, this returns `()`. If `Self = usize`, this returns + /// `elems`. No other types are currently supported. + fn from_elem_count(elems: usize) -> Self; + + /// Converts `self` to an element count. + /// + /// If `Self = ()`, this returns `0`. If `Self = usize`, this returns + /// `self`. No other types are currently supported. + fn to_elem_count(self) -> usize; + + /// Computes the size of the object with the given layout and pointer + /// metadata. + /// + /// # Panics + /// + /// If `Self = ()`, `layout` must describe a sized type. If `Self = usize`, + /// `layout` must describe a slice DST. Otherwise, `size_for_metadata` may + /// panic. + /// + /// # Safety + /// + /// `size_for_metadata` promises to only return `None` if the resulting size + /// would not fit in a `usize`. + fn size_for_metadata(self, layout: DstLayout) -> Option; +} + +impl PointerMetadata for () { + #[inline] + #[allow(clippy::unused_unit)] + fn from_elem_count(_elems: usize) -> () {} + + #[inline] + fn to_elem_count(self) -> usize { + 0 + } + + #[inline] + fn size_for_metadata(self, layout: DstLayout) -> Option { + match layout.size_info { + SizeInfo::Sized { size } => Some(size), + // NOTE: This branch is unreachable, but we return `None` rather + // than `unreachable!()` to avoid generating panic paths. + SizeInfo::SliceDst(_) => None, + } + } +} + +impl PointerMetadata for usize { + #[inline] + fn from_elem_count(elems: usize) -> usize { + elems + } + + #[inline] + fn to_elem_count(self) -> usize { + self + } + + #[inline] + fn size_for_metadata(self, layout: DstLayout) -> Option { + match layout.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + let slice_len = elem_size.checked_mul(self)?; + let without_padding = offset.checked_add(slice_len)?; + without_padding.checked_add(util::padding_needed_for(without_padding, layout.align)) + } + // NOTE: This branch is unreachable, but we return `None` rather + // than `unreachable!()` to avoid generating panic paths. + SizeInfo::Sized { .. } => None, + } + } +} + +// SAFETY: Delegates safety to `DstLayout::for_slice`. +unsafe impl KnownLayout for [T] { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type PointerMetadata = usize; + + // SAFETY: `CoreMaybeUninit::LAYOUT` and `T::LAYOUT` are identical + // because `CoreMaybeUninit` has the same size and alignment as `T` [1]. + // Consequently, `[CoreMaybeUninit]::LAYOUT` and `[T]::LAYOUT` are + // identical, because they both lack a fixed-sized prefix and because they + // inherit the alignments of their inner element type (which are identical) + // [2][3]. + // + // `[CoreMaybeUninit]` admits uninitialized bytes at all positions + // because `CoreMaybeUninit` admits uninitialized bytes at all positions + // and because the inner elements of `[CoreMaybeUninit]` are laid out + // back-to-back [2][3]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit` is guaranteed to have the same size, alignment, and ABI as + // `T` + // + // [2] Per https://doc.rust-lang.org/1.82.0/reference/type-layout.html#slice-layout: + // + // Slices have the same layout as the section of the array they slice. + // + // [3] Per https://doc.rust-lang.org/1.82.0/reference/type-layout.html#array-layout: + // + // An array of `[T; N]` has a size of `size_of::() * N` and the same + // alignment of `T`. Arrays are laid out so that the zero-based `nth` + // element of the array is offset from the start of the array by `n * + // size_of::()` bytes. + type MaybeUninit = [CoreMaybeUninit]; + + const LAYOUT: DstLayout = DstLayout::for_slice::(); + + // SAFETY: `.cast` preserves address and provenance. The returned pointer + // refers to an object with `elems` elements by construction. + #[inline(always)] + fn raw_from_ptr_len(data: NonNull, elems: usize) -> NonNull { + // FIXME(#67): Remove this allow. See NonNullExt for more details. + #[allow(unstable_name_collisions)] + NonNull::slice_from_raw_parts(data.cast::(), elems) + } + + #[inline(always)] + fn pointer_to_metadata(ptr: *mut [T]) -> usize { + #[allow(clippy::as_conversions)] + let slc = ptr as *const [()]; + + // SAFETY: + // - `()` has alignment 1, so `slc` is trivially aligned. + // - `slc` was derived from a non-null pointer. + // - The size is 0 regardless of the length, so it is sound to + // materialize a reference regardless of location. + // - By invariant, `self.ptr` has valid provenance. + let slc = unsafe { &*slc }; + + // This is correct because the preceding `as` cast preserves the number + // of slice elements. [1] + // + // [1] Per https://doc.rust-lang.org/reference/expressions/operator-expr.html#pointer-to-pointer-cast: + // + // For slice types like `[T]` and `[U]`, the raw pointer types `*const + // [T]`, `*mut [T]`, `*const [U]`, and `*mut [U]` encode the number of + // elements in this slice. Casts between these raw pointer types + // preserve the number of elements. ... The same holds for `str` and + // any compound type whose unsized tail is a slice type, such as + // struct `Foo(i32, [u8])` or `(u64, Foo)`. + slc.len() + } +} + +#[rustfmt::skip] +impl_known_layout!( + (), + u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize, f32, f64, + bool, char, + NonZeroU8, NonZeroI8, NonZeroU16, NonZeroI16, NonZeroU32, NonZeroI32, + NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize +); +#[rustfmt::skip] +#[cfg(feature = "float-nightly")] +impl_known_layout!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] + f16, + #[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] + f128 +); +#[rustfmt::skip] +impl_known_layout!( + T => Option, + T: ?Sized => PhantomData, + T => Wrapping, + T => CoreMaybeUninit, + T: ?Sized => *const T, + T: ?Sized => *mut T, + T: ?Sized => &'_ T, + T: ?Sized => &'_ mut T, +); +impl_known_layout!(const N: usize, T => [T; N]); + +// SAFETY: `str` has the same representation as `[u8]`. `ManuallyDrop` [1], +// `UnsafeCell` [2], and `Cell` [3] have the same representation as `T`. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop` is guaranteed to have the same layout and bit validity as +// `T` +// +// [2] Per https://doc.rust-lang.org/1.85.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell` has the same in-memory representation as its inner type +// `T`. +// +// [3] Per https://doc.rust-lang.org/1.85.0/core/cell/struct.Cell.html#memory-layout: +// +// `Cell` has the same in-memory representation as `T`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl_known_layout!( + #[repr([u8])] + str + ); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] ManuallyDrop); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] UnsafeCell); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] Cell); +}; + +// SAFETY: +// - By consequence of the invariant on `T::MaybeUninit` that `T::LAYOUT` and +// `T::MaybeUninit::LAYOUT` are equal, `T` and `T::MaybeUninit` have the same: +// - Fixed prefix size +// - Alignment +// - (For DSTs) trailing slice element size +// - By consequence of the above, referents `T::MaybeUninit` and `T` have the +// require the same kind of pointer metadata, and thus it is valid to perform +// an `as` cast from `*mut T` and `*mut T::MaybeUninit`, and this operation +// preserves referent size (ie, `size_of_val_raw`). +const _: () = unsafe { + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T::MaybeUninit)] MaybeUninit) +}; + +// FIXME(#196, #2856): Eventually, we'll want to support enums variants and +// union fields being treated uniformly since they behave similarly to each +// other in terms of projecting validity – specifically, for a type `T` with +// validity `V`, if `T` is a struct type, then its fields straightforwardly also +// have validity `V`. By contrast, if `T` is an enum or union type, then +// validity is not straightforwardly recursive in this way. +#[doc(hidden)] +pub const STRUCT_VARIANT_ID: i128 = -1; +#[doc(hidden)] +pub const UNION_VARIANT_ID: i128 = -2; +#[doc(hidden)] +pub const REPR_C_UNION_VARIANT_ID: i128 = -3; + +/// # Safety +/// +/// `Self::ProjectToTag` must satisfy its safety invariant. +#[doc(hidden)] +pub unsafe trait HasTag { + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type's enum tag, or `()` for non-enum types. + type Tag: Immutable; + + /// A pointer projection from `Self` to its tag. + /// + /// # Safety + /// + /// It must be the case that, for all `slf: Ptr<'_, Self, I>`, it is sound + /// to project from `slf` to `Ptr<'_, Self::Tag, I>` using this projection. + type ProjectToTag: pointer::cast::Project; +} + +/// Projects a given field from `Self`. +/// +/// All implementations of `HasField` for a particular field `f` in `Self` +/// should use the same `Field` type; this ensures that `Field` is inferable +/// given an explicit `VARIANT_ID` and `FIELD_ID`. +/// +/// # Safety +/// +/// A field `f` is `HasField` for `Self` if and only if: +/// +/// - If `Self` has the layout of a struct or union type, then `VARIANT_ID` is +/// `STRUCT_VARIANT_ID` or `UNION_VARIANT_ID` respectively; otherwise, if +/// `Self` has the layout of an enum type, `VARIANT_ID` is the numerical index +/// of the enum variant in which `f` appears. Note that `Self` does not need +/// to actually *be* such a type – it just needs to have the same layout as +/// such a type. For example, a `#[repr(transparent)]` wrapper around an enum +/// has the same layout as that enum. +/// - If `f` has name `n`, `FIELD_ID` is `zerocopy::ident_id!(n)`; otherwise, +/// if `f` is at index `i`, `FIELD_ID` is `zerocopy::ident_id!(i)`. +/// - `Field` is a type with the same visibility as `f`. +/// - `Type` has the same type as `f`. +/// +/// The caller must **not** assume that a pointer's referent being aligned +/// implies that calling `project` on that pointer will result in a pointer to +/// an aligned referent. For example, `HasField` may be implemented for +/// `#[repr(packed)]` structs. +/// +/// The implementation of `project` must satisfy its safety post-condition. +#[doc(hidden)] +pub unsafe trait HasField: + HasTag +{ + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type of the field. + type Type: ?Sized; + + /// Projects from `slf` to the field. + /// + /// Users should generally not call `project` directly, and instead should + /// use high-level APIs like [`PtrInner::project`] or [`Ptr::project`]. + /// + /// # Safety + /// + /// The returned pointer refers to a non-strict subset of the bytes of + /// `slf`'s referent, and has the same provenance as `slf`. + #[must_use] + fn project(slf: PtrInner<'_, Self>) -> *mut Self::Type; +} + +/// Projects a given field from `Self`. +/// +/// Implementations of this trait encode the conditions under which a field can +/// be projected from a `Ptr<'_, Self, I>`, and how the invariants of that +/// [`Ptr`] (`I`) determine the invariants of pointers projected from it. In +/// other words, it is a type-level function over invariants; `I` goes in, +/// `Self::Invariants` comes out. +/// +/// # Safety +/// +/// `T: ProjectField` if, for a +/// `ptr: Ptr<'_, T, I>` such that `T::is_projectable(ptr).is_ok()`, +/// `>::project(ptr.as_inner())` +/// conforms to `T::Invariants`. +#[doc(hidden)] +pub unsafe trait ProjectField: + HasField +where + I: invariant::Invariants, +{ + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The invariants of the projected field pointer, with respect to the + /// invariants, `I`, of the containing pointer. The aliasing dimension of + /// the invariants is guaranteed to remain unchanged. + type Invariants: invariant::Invariants; + + /// The failure mode of projection. `()` if the projection is fallible, + /// otherwise [`core::convert::Infallible`]. + type Error; + + /// Is the given field projectable from `ptr`? + /// + /// If a field with [`Self::Invariants`] is projectable from the referent, + /// this function produces an `Ok(ptr)` from which the projection can be + /// made; otherwise `Err`. + /// + /// This method must be overriden if the field's projectability depends on + /// the value of the bytes in `ptr`. + #[inline(always)] + fn is_projectable<'a>(_ptr: Ptr<'a, Self::Tag, I>) -> Result<(), Self::Error> { + trait IsInfallible { + const IS_INFALLIBLE: bool; + } + + struct Projection( + PhantomData<(Field, I, T)>, + ) + where + T: ?Sized + HasField, + I: invariant::Invariants; + + impl IsInfallible + for Projection + where + T: ?Sized + HasField, + I: invariant::Invariants, + { + const IS_INFALLIBLE: bool = { + let is_infallible = match VARIANT_ID { + // For nondestructive projections of struct and union + // fields, the projected field's satisfaction of + // `Invariants` does not depend on the value of the + // referent. This default implementation of `is_projectable` + // is non-destructive, as it does not overwrite any part of + // the referent. + crate::STRUCT_VARIANT_ID | crate::UNION_VARIANT_ID => true, + _enum_variant => { + use crate::invariant::{Validity, ValidityKind}; + match I::Validity::KIND { + // The `Uninit` and `Initialized` validity + // invariants do not depend on the enum's tag. In + // particular, we don't actually care about what + // variant is present – we can treat *any* range of + // uninitialized or initialized memory as containing + // an uninitialized or initialized instance of *any* + // type – the type itself is irrelevant. + ValidityKind::Uninit | ValidityKind::Initialized => true, + // The projectability of an enum field from an + // `AsInitialized` or `Valid` state is a dynamic + // property of its tag. + ValidityKind::AsInitialized | ValidityKind::Valid => false, + } + } + }; + const_assert!(is_infallible); + is_infallible + }; + } + + const_assert!( + as IsInfallible>::IS_INFALLIBLE + ); + + Ok(()) + } +} + +/// Analyzes whether a type is [`FromZeros`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `FromZeros` and implements `FromZeros` and its +/// supertraits if it is sound to do so. This derive can be applied to structs, +/// enums, and unions; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromZeros, Immutable}; +/// #[derive(FromZeros)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@FromZeros#safety +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `FromZeros` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `FromZeros` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `FromZeros` for that type: +/// +/// - If the type is a struct, all of its fields must be `FromZeros`. +/// - If the type is an enum: +/// - It must have a defined representation (`repr`s `C`, `u8`, `u16`, `u32`, +/// `u64`, `usize`, `i8`, `i16`, `i32`, `i64`, or `isize`). +/// - It must have a variant with a discriminant/tag of `0`, and its fields +/// must be `FromZeros`. See [the reference] for a description of +/// discriminant values are specified. +/// - The fields of that variant must be `FromZeros`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromZeros`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [the reference]: https://doc.rust-lang.org/reference/items/enumerations.html#custom-discriminant-values-for-fieldless-enumerations +/// +/// ## Why isn't an explicit representation required for structs? +/// +/// Neither this derive, nor the [safety conditions] of `FromZeros`, requires +/// that structs are marked with `#[repr(C)]`. +/// +/// Per the [Rust reference](reference), +/// +/// > The representation of a type can change the padding between fields, but +/// > does not change the layout of the fields themselves. +/// +/// [reference]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// +/// Since the layout of structs only consists of padding bytes and field bytes, +/// a struct is soundly `FromZeros` if: +/// 1. its padding is soundly `FromZeros`, and +/// 2. its fields are soundly `FromZeros`. +/// +/// The answer to the first question is always yes: padding bytes do not have +/// any validity constraints. A [discussion] of this question in the Unsafe Code +/// Guidelines Working Group concluded that it would be virtually unimaginable +/// for future versions of rustc to add validity constraints to padding bytes. +/// +/// [discussion]: https://github.com/rust-lang/unsafe-code-guidelines/issues/174 +/// +/// Whether a struct is soundly `FromZeros` therefore solely depends on whether +/// its fields are `FromZeros`. +// FIXME(#146): Document why we don't require an enum to have an explicit `repr` +// attribute. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::FromZeros; +/// Analyzes whether a type is [`Immutable`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `Immutable` and implements `Immutable` if it is +/// sound to do so. This derive can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Immutable; +/// #[derive(Immutable)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `Immutable` for a given type. +/// Unless you are modifying the implementation of this derive, you don't need +/// to read this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `Immutable` for that type: +/// +/// - All fields must be `Immutable`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `Immutable`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [safety conditions]: trait@Immutable#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::Immutable; + +/// Types which are free from interior mutability. +/// +/// `T: Immutable` indicates that `T` does not permit interior mutation, except +/// by ownership or an exclusive (`&mut`) borrow. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(Immutable)]`][derive] (requires the `derive` Cargo feature); +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Immutable; +/// #[derive(Immutable)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// enum MyEnum { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `Immutable`. +/// +/// # Safety +/// +/// Unsafe code outside of this crate must not make any assumptions about `T` +/// based on `T: Immutable`. We reserve the right to relax the requirements for +/// `Immutable` in the future, and if unsafe code outside of this crate makes +/// assumptions based on `T: Immutable`, future relaxations may cause that code +/// to become unsound. +/// +// # Safety (Internal) +// +// If `T: Immutable`, unsafe code *inside of this crate* may assume that, given +// `t: &T`, `t` does not permit interior mutation of its referent. Because +// [`UnsafeCell`] is the only type which permits interior mutation, it is +// sufficient (though not necessary) to guarantee that `T` contains no +// `UnsafeCell`s. +// +// [`UnsafeCell`]: core::cell::UnsafeCell +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::Immutable", + doc = "[derive-analysis]: zerocopy_derive::Immutable#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Immutable.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Immutable.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(Immutable)]` to `{Self}`") +)] +pub unsafe trait Immutable { + // The `Self: Sized` bound makes it so that `Immutable` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; +} + +/// Implements [`TryFromBytes`]. +/// +/// This derive synthesizes the runtime checks required to check whether a +/// sequence of initialized bytes corresponds to a valid instance of a type. +/// This derive can be applied to structs, enums, and unions; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{TryFromBytes, Immutable}; +/// #[derive(TryFromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Portability +/// +/// To ensure consistent endianness for enums with multi-byte representations, +/// explicitly specify and convert each discriminant using `.to_le()` or +/// `.to_be()`; e.g.: +/// +/// ``` +/// # use zerocopy_derive::TryFromBytes; +/// // `DataStoreVersion` is encoded in little-endian. +/// #[derive(TryFromBytes)] +/// #[repr(u32)] +/// pub enum DataStoreVersion { +/// /// Version 1 of the data store. +/// V1 = 9u32.to_le(), +/// +/// /// Version 2 of the data store. +/// V2 = 10u32.to_le(), +/// } +/// ``` +/// +/// [safety conditions]: trait@TryFromBytes#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::TryFromBytes; + +/// Types for which some bit patterns are valid. +/// +/// A memory region of the appropriate length which contains initialized bytes +/// can be viewed as a `TryFromBytes` type so long as the runtime value of those +/// bytes corresponds to a [*valid instance*] of that type. For example, +/// [`bool`] is `TryFromBytes`, so zerocopy can transmute a [`u8`] into a +/// [`bool`] so long as it first checks that the value of the [`u8`] is `0` or +/// `1`. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(TryFromBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{TryFromBytes, Immutable}; +/// #[derive(TryFromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive ensures that the runtime check of whether bytes correspond to a +/// valid instance is sound. You **must** implement this trait via the derive. +/// +/// # What is a "valid instance"? +/// +/// In Rust, each type has *bit validity*, which refers to the set of bit +/// patterns which may appear in an instance of that type. It is impossible for +/// safe Rust code to produce values which violate bit validity (ie, values +/// outside of the "valid" set of bit patterns). If `unsafe` code produces an +/// invalid value, this is considered [undefined behavior]. +/// +/// Rust's bit validity rules are currently being decided, which means that some +/// types have three classes of bit patterns: those which are definitely valid, +/// and whose validity is documented in the language; those which may or may not +/// be considered valid at some point in the future; and those which are +/// definitely invalid. +/// +/// Zerocopy takes a conservative approach, and only considers a bit pattern to +/// be valid if its validity is a documented guarantee provided by the +/// language. +/// +/// For most use cases, Rust's current guarantees align with programmers' +/// intuitions about what ought to be valid. As a result, zerocopy's +/// conservatism should not affect most users. +/// +/// If you are negatively affected by lack of support for a particular type, +/// we encourage you to let us know by [filing an issue][github-repo]. +/// +/// # `TryFromBytes` is not symmetrical with [`IntoBytes`] +/// +/// There are some types which implement both `TryFromBytes` and [`IntoBytes`], +/// but for which `TryFromBytes` is not guaranteed to accept all byte sequences +/// produced by `IntoBytes`. In other words, for some `T: TryFromBytes + +/// IntoBytes`, there exist values of `t: T` such that +/// `TryFromBytes::try_ref_from_bytes(t.as_bytes()) == None`. Code should not +/// generally assume that values produced by `IntoBytes` will necessarily be +/// accepted as valid by `TryFromBytes`. +/// +/// # Safety +/// +/// On its own, `T: TryFromBytes` does not make any guarantees about the layout +/// or representation of `T`. It merely provides the ability to perform a +/// validity check at runtime via methods like [`try_ref_from_bytes`]. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `TryFromBytes`. +/// Future releases of zerocopy may make backwards-breaking changes to these +/// items, including changes that only affect soundness, which may cause code +/// which uses those items to silently become unsound. +/// +/// [undefined behavior]: https://raphlinus.github.io/programming/rust/2018/08/17/undefined-behavior.html +/// [github-repo]: https://github.com/google/zerocopy +/// [`try_ref_from_bytes`]: TryFromBytes::try_ref_from_bytes +/// [*valid instance*]: #what-is-a-valid-instance +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::TryFromBytes")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.TryFromBytes.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(TryFromBytes)]` to `{Self}`") +)] +pub unsafe trait TryFromBytes { + // The `Self: Sized` bound makes it so that `TryFromBytes` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Does a given memory range contain a valid instance of `Self`? + /// + /// # Safety + /// + /// Unsafe code may assume that, if `is_bit_valid(candidate)` returns true, + /// `*candidate` contains a valid `Self`. + /// + /// # Panics + /// + /// `is_bit_valid` may panic. Callers are responsible for ensuring that any + /// `unsafe` code remains sound even in the face of `is_bit_valid` + /// panicking. (We support user-defined validation routines; so long as + /// these routines are not required to be `unsafe`, there is no way to + /// ensure that these do not generate panics.) + /// + /// Besides user-defined validation routines panicking, `is_bit_valid` will + /// either panic or fail to compile if called on a pointer with [`Shared`] + /// aliasing when `Self: !Immutable`. + /// + /// [`UnsafeCell`]: core::cell::UnsafeCell + /// [`Shared`]: invariant::Shared + #[doc(hidden)] + fn is_bit_valid(candidate: Maybe<'_, Self, A>) -> bool + where + A: invariant::Alignment; + + /// Attempts to interpret the given `source` as a `&Self`. + /// + /// If the bytes of `source` are a valid instance of `Self`, this method + /// returns a reference to those bytes interpreted as a `Self`. If the + /// length of `source` is not a [valid size of `Self`][valid-size], or if + /// `source` is not appropriately aligned, or if `source` is not a valid + /// instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_bytes(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the byte sequence `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// + /// let packet = Packet::try_ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// assert!(Packet::try_ref_from_bytes(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_bytes(source: &[u8]) -> Result<&Self, TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_ref(source).try_cast_into_no_leftover::(None) { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(valid) => Ok(valid.as_ref()), + Err(e) => { + Err(e.map_src(|src| src.as_bytes::().as_ref()).into()) + } + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`. If that prefix is a valid + /// instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, or + /// if those bytes are not a valid instance of `Self`, this returns `Err`. + /// If [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_prefix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_ref_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// assert_eq!(suffix, &[6u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_ref_from_prefix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_prefix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_prefix(source: &[u8]) -> Result<(&Self, &[u8]), TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + try_ref_from_prefix_suffix(source, CastType::Prefix, None) + } + + /// Attempts to interpret the suffix of the given `source` as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`. If that suffix is a + /// valid instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if the suffix of `source` would not be + /// appropriately aligned, or if the suffix is not a valid instance of + /// `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], you + /// can [infallibly discard the alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_suffix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_ref_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[0u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0x10][..]; + /// assert!(Packet::try_ref_from_suffix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_suffix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_suffix(source: &[u8]) -> Result<(&[u8], &Self), TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + try_ref_from_prefix_suffix(source, CastType::Suffix, None).map(swap) + } + + /// Attempts to interpret the given `source` as a `&mut Self` without + /// copying. + /// + /// If the bytes of `source` are a valid instance of `Self`, this method + /// returns a reference to those bytes interpreted as a `Self`. If the + /// length of `source` is not a [valid size of `Self`][valid-size], or if + /// `source` is not appropriately aligned, or if `source` is not a valid + /// instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_bytes(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// + /// let packet = Packet::try_mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// + /// packet.temperature = 111; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 0, 1, 2, 3, 4, 5]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_mut_from_bytes(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_bytes")] + /// + /// See [`TryFromBytes::try_ref_from_bytes`](#method.try_ref_from_bytes.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_bytes(bytes: &mut [u8]) -> Result<&mut Self, TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_mut(bytes).try_cast_into_no_leftover::(None) { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_mut()), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&mut + /// Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`. If that prefix is a valid + /// instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, or + /// if the bytes are not a valid instance of `Self`, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_prefix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_mut_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// assert_eq!(suffix, &[6u8][..]); + /// + /// packet.temperature = 111; + /// suffix[0] = 222; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 0, 1, 2, 3, 4, 5, 222]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_mut_from_prefix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_prefix")] + /// + /// See [`TryFromBytes::try_ref_from_prefix`](#method.try_ref_from_prefix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_prefix( + source: &mut [u8], + ) -> Result<(&mut Self, &mut [u8]), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + try_mut_from_prefix_suffix(source, CastType::Prefix, None) + } + + /// Attempts to interpret the suffix of the given `source` as a `&mut + /// Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`. If that suffix is a + /// valid instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if the suffix of `source` would not be + /// appropriately aligned, or if the suffix is not a valid instance of + /// `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], you + /// can [infallibly discard the alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_suffix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &mut [0, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_mut_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[0u8][..]); + /// + /// prefix[0] = 111; + /// packet.temperature = 222; + /// + /// assert_eq!(bytes, [111, 0xC0, 0xC0, 240, 222, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0x10][..]; + /// assert!(Packet::try_mut_from_suffix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_suffix")] + /// + /// See [`TryFromBytes::try_ref_from_suffix`](#method.try_ref_from_suffix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_suffix( + source: &mut [u8], + ) -> Result<(&mut [u8], &mut Self), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + try_mut_from_prefix_suffix(source, CastType::Suffix, None).map(swap) + } + + /// Attempts to interpret the given `source` as a `&Self` with a DST length + /// equal to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, if `source` is not + /// appropriately aligned, or if `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let packet = Packet::try_ref_from_bytes_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_bytes_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let zsty = ZSTy::try_ref_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_bytes`]: TryFromBytes::try_ref_from_bytes + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_bytes_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_bytes_with_elems( + source: &[u8], + count: usize, + ) -> Result<&Self, TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + match Ptr::from_ref(source).try_cast_into_no_leftover::(Some(count)) + { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_ref()), + Err(e) => { + Err(e.map_src(|src| src.as_bytes::().as_ref()).into()) + } + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&Self` with + /// a DST length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if `source` is not appropriately + /// aligned, or if the prefix of `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7, 8][..]; + /// + /// let (packet, suffix) = Packet::try_ref_from_prefix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(suffix, &[8u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_prefix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let (zsty, _) = ZSTy::try_ref_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_prefix`]: TryFromBytes::try_ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_prefix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_prefix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&Self, &[u8]), TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + try_ref_from_prefix_suffix(source, CastType::Prefix, Some(count)) + } + + /// Attempts to interpret the suffix of the given `source` as a `&Self` with + /// a DST length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if the suffix of `source` is not + /// appropriately aligned, or if the suffix of `source` does not contain a + /// valid instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[123, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_ref_from_suffix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[123u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_suffix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let (_, zsty) = ZSTy::try_ref_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_prefix`]: TryFromBytes::try_ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_suffix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_suffix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&[u8], &Self), TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + try_ref_from_prefix_suffix(source, CastType::Suffix, Some(count)).map(swap) + } + + /// Attempts to interpret the given `source` as a `&mut Self` with a DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, if `source` is not + /// appropriately aligned, or if `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let packet = Packet::try_mut_from_bytes_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// + /// packet.temperature = 111; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_bytes_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let zsty = ZSTy::try_mut_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_bytes`]: TryFromBytes::try_mut_from_bytes + /// + #[doc = codegen_header!("h5", "try_mut_from_bytes_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_bytes_with_elems`](#method.try_ref_from_bytes_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_bytes_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<&mut Self, TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + match Ptr::from_mut(source).try_cast_into_no_leftover::(Some(count)) + { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_mut()), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&mut Self` + /// with a DST length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if `source` is not appropriately + /// aligned, or if the prefix of `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7, 8][..]; + /// + /// let (packet, suffix) = Packet::try_mut_from_prefix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(suffix, &[8u8][..]); + /// + /// packet.temperature = 111; + /// suffix[0] = 222; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 2, 3, 4, 5, 6, 7, 222]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_prefix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let (zsty, _) = ZSTy::try_mut_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_prefix`]: TryFromBytes::try_mut_from_prefix + /// + #[doc = codegen_header!("h5", "try_mut_from_prefix_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_prefix_with_elems`](#method.try_ref_from_prefix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_prefix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut Self, &mut [u8]), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + try_mut_from_prefix_suffix(source, CastType::Prefix, Some(count)) + } + + /// Attempts to interpret the suffix of the given `source` as a `&mut Self` + /// with a DST length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if the suffix of `source` is not + /// appropriately aligned, or if the suffix of `source` does not contain a + /// valid instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [123, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_mut_from_suffix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[123u8][..]); + /// + /// prefix[0] = 111; + /// packet.temperature = 222; + /// + /// assert_eq!(bytes, [111, 0xC0, 0xC0, 240, 222, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_suffix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let (_, zsty) = ZSTy::try_mut_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_prefix`]: TryFromBytes::try_mut_from_prefix + /// + #[doc = codegen_header!("h5", "try_mut_from_suffix_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_suffix_with_elems`](#method.try_ref_from_suffix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_suffix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut [u8], &mut Self), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + try_mut_from_prefix_suffix(source, CastType::Suffix, Some(count)).map(swap) + } + + /// Attempts to read the given `source` as a `Self`. + /// + /// If `source.len() != size_of::()` or the bytes are not a valid + /// instance of `Self`, this returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77][..]; + /// + /// let packet = Packet::try_read_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77][..]; + /// assert!(Packet::try_read_from_bytes(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_bytes", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_bytes(source: &[u8]) -> Result> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::() == 1`, validate `source` in-place. + + let candidate = match CoreMaybeUninit::::read_from_bytes(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate) } + } + + /// Attempts to read a `Self` from the prefix of the given `source`. + /// + /// This attempts to read a `Self` from the first `size_of::()` bytes + /// of `source`, returning that `Self` and any remaining bytes. If + /// `source.len() < size_of::()` or the bytes are not a valid instance + /// of `Self`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_read_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(suffix, &[0u8, 1, 2, 3, 4, 5, 6][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_read_from_prefix(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_prefix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_prefix(source: &[u8]) -> Result<(Self, &[u8]), TryReadError<&[u8], Self>> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::() == 1`, validate `source` in-place. + + let (candidate, suffix) = match CoreMaybeUninit::::read_from_prefix(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate).map(|slf| (slf, suffix)) } + } + + /// Attempts to read a `Self` from the suffix of the given `source`. + /// + /// This attempts to read a `Self` from the last `size_of::()` bytes + /// of `source`, returning that `Self` and any preceding bytes. If + /// `source.len() < size_of::()` or the bytes are not a valid instance + /// of `Self`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 0xC0, 0xC0, 240, 77][..]; + /// + /// let (prefix, packet) = Packet::try_read_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(prefix, &[0u8, 1, 2, 3, 4, 5][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 0x10, 0xC0, 240, 77][..]; + /// assert!(Packet::try_read_from_suffix(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_suffix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_suffix(source: &[u8]) -> Result<(&[u8], Self), TryReadError<&[u8], Self>> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::() == 1`, validate `source` in-place. + + let (prefix, candidate) = match CoreMaybeUninit::::read_from_suffix(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate).map(|slf| (prefix, slf)) } + } +} + +#[inline(always)] +fn try_ref_from_prefix_suffix( + source: &[u8], + cast_type: CastType, + meta: Option, +) -> Result<(&T, &[u8]), TryCastError<&[u8], T>> { + match Ptr::from_ref(source).try_cast_into::(cast_type, meta) { + Ok((source, prefix_suffix)) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(valid) => Ok((valid.as_ref(), prefix_suffix.as_ref())), + Err(e) => Err(e.map_src(|src| src.as_bytes::().as_ref()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } +} + +#[inline(always)] +fn try_mut_from_prefix_suffix( + candidate: &mut [u8], + cast_type: CastType, + meta: Option, +) -> Result<(&mut T, &mut [u8]), TryCastError<&mut [u8], T>> { + match Ptr::from_mut(candidate).try_cast_into::(cast_type, meta) { + Ok((candidate, prefix_suffix)) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match candidate.try_into_valid() { + Ok(valid) => Ok((valid.as_mut(), prefix_suffix.as_mut())), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } +} + +#[inline(always)] +fn swap((t, u): (T, U)) -> (U, T) { + (u, t) +} + +/// # Safety +/// +/// All bytes of `candidate` must be initialized. +#[inline(always)] +unsafe fn try_read_from( + source: S, + mut candidate: CoreMaybeUninit, +) -> Result> { + // We use `from_mut` despite not mutating via `c_ptr` so that we don't need + // to add a `T: Immutable` bound. + let c_ptr = Ptr::from_mut(&mut candidate); + // SAFETY: `c_ptr` has no uninitialized sub-ranges because it derived from + // `candidate`, which the caller promises is entirely initialized. Since + // `candidate` is a `MaybeUninit`, it has no validity requirements, and so + // no values written to an `Initialized` `c_ptr` can violate its validity. + // Since `c_ptr` has `Exclusive` aliasing, no mutations may happen except + // via `c_ptr` so long as it is live, so we don't need to worry about the + // fact that `c_ptr` may have more restricted validity than `candidate`. + let c_ptr = unsafe { c_ptr.assume_validity::() }; + let mut c_ptr = c_ptr.cast::<_, crate::pointer::cast::CastSized, _>(); + + // Since we don't have `T: KnownLayout`, we hack around that by using + // `Wrapping`, which implements `KnownLayout` even if `T` doesn't. + // + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to fix + // before returning. + if !Wrapping::::is_bit_valid(c_ptr.reborrow_shared().forget_aligned()) { + return Err(ValidityError::new(source).into()); + } + + fn _assert_same_size_and_validity() + where + Wrapping: pointer::TransmuteFrom, + T: pointer::TransmuteFrom, invariant::Valid, invariant::Valid>, + { + } + + _assert_same_size_and_validity::(); + + // SAFETY: We just validated that `candidate` contains a valid + // `Wrapping`, which has the same size and bit validity as `T`, as + // guaranteed by the preceding type assertion. + Ok(unsafe { candidate.assume_init() }) +} + +/// Types for which a sequence of `0` bytes is a valid instance. +/// +/// Any memory region of the appropriate length which is guaranteed to contain +/// only zero bytes can be viewed as any `FromZeros` type with no runtime +/// overhead. This is useful whenever memory is known to be in a zeroed state, +/// such memory returned from some allocation routines. +/// +/// # Warning: Padding bytes +/// +/// Note that, when a value is moved or copied, only the non-padding bytes of +/// that value are guaranteed to be preserved. It is unsound to assume that +/// values written to padding bytes are preserved after a move or copy. For more +/// details, see the [`FromBytes` docs][frombytes-warning-padding-bytes]. +/// +/// [frombytes-warning-padding-bytes]: FromBytes#warning-padding-bytes +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(FromZeros)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromZeros, Immutable}; +/// #[derive(FromZeros)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `FromZeros`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: FromZeros`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `FromZeros` manually, and you don't plan on writing unsafe code that +/// operates on `FromZeros` types, then you don't need to read this section.* +/// +/// If `T: FromZeros`, then unsafe code may assume that it is sound to produce a +/// `T` whose bytes are all initialized to zero. If a type is marked as +/// `FromZeros` which violates this contract, it may cause undefined behavior. +/// +/// `#[derive(FromZeros)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::FromZeros", + doc = "[derive-analysis]: zerocopy_derive::FromZeros#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromZeros.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromZeros.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(FromZeros)]` to `{Self}`") +)] +pub unsafe trait FromZeros: TryFromBytes { + // The `Self: Sized` bound makes it so that `FromZeros` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Overwrites `self` with zeros. + /// + /// Sets every byte in `self` to 0. While this is similar to doing `*self = + /// Self::new_zeroed()`, it differs in that `zero` does not semantically + /// drop the current value and replace it with a new one — it simply + /// modifies the bytes of the existing value. + /// + /// # Examples + /// + /// ``` + /// # use zerocopy::FromZeros; + /// # use zerocopy_derive::*; + /// # + /// #[derive(FromZeros)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let mut header = PacketHeader { + /// src_port: 100u16.to_be_bytes(), + /// dst_port: 200u16.to_be_bytes(), + /// length: 300u16.to_be_bytes(), + /// checksum: 400u16.to_be_bytes(), + /// }; + /// + /// header.zero(); + /// + /// assert_eq!(header.src_port, [0, 0]); + /// assert_eq!(header.dst_port, [0, 0]); + /// assert_eq!(header.length, [0, 0]); + /// assert_eq!(header.checksum, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "zero", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline(always)] + fn zero(&mut self) { + let slf: *mut Self = self; + let len = mem::size_of_val(self); + // SAFETY: + // - `self` is guaranteed by the type system to be valid for writes of + // size `size_of_val(self)`. + // - `u8`'s alignment is 1, and thus `self` is guaranteed to be aligned + // as required by `u8`. + // - Since `Self: FromZeros`, the all-zeros instance is a valid instance + // of `Self.` + // + // FIXME(#429): Add references to docs and quotes. + unsafe { ptr::write_bytes(slf.cast::(), 0, len) }; + } + + /// Creates an instance of `Self` from zeroed bytes. + /// + /// # Examples + /// + /// ``` + /// # use zerocopy::FromZeros; + /// # use zerocopy_derive::*; + /// # + /// #[derive(FromZeros)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header: PacketHeader = FromZeros::new_zeroed(); + /// + /// assert_eq!(header.src_port, [0, 0]); + /// assert_eq!(header.dst_port, [0, 0]); + /// assert_eq!(header.length, [0, 0]); + /// assert_eq!(header.checksum, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[inline(always)] + fn new_zeroed() -> Self + where + Self: Sized, + { + // SAFETY: `FromZeros` says that the all-zeros bit pattern is legal. + unsafe { mem::zeroed() } + } + + /// Creates a `Box` from zeroed bytes. + /// + /// This function is useful for allocating large values on the heap and + /// zero-initializing them, without ever creating a temporary instance of + /// `Self` on the stack. For example, `<[u8; 1048576]>::new_box_zeroed()` + /// will allocate `[u8; 1048576]` directly on the heap; it does not require + /// storing `[u8; 1048576]` in a temporary variable on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_box_zeroed` (or related functions) may + /// have performance benefits. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is guaranteed + /// never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_box_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(any(feature = "alloc", test))] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline] + fn new_box_zeroed() -> Result, AllocError> + where + Self: Sized, + { + // If `T` is a ZST, then return a proper boxed instance of it. There is + // no allocation, but `Box` does require a correct dangling pointer. + let layout = Layout::new::(); + if layout.size() == 0 { + // Construct the `Box` from a dangling pointer to avoid calling + // `Self::new_zeroed`. This ensures that stack space is never + // allocated for `Self` even on lower opt-levels where this branch + // might not get optimized out. + + // SAFETY: Per [1], when `T` is a ZST, `Box`'s only validity + // requirements are that the pointer is non-null and sufficiently + // aligned. Per [2], `NonNull::dangling` produces a pointer which + // is sufficiently aligned. Since the produced pointer is a + // `NonNull`, it is non-null. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/boxed/index.html#memory-layout: + // + // For zero-sized values, the `Box` pointer has to be non-null and sufficiently aligned. + // + // [2] Per https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.dangling: + // + // Creates a new `NonNull` that is dangling, but well-aligned. + return Ok(unsafe { Box::from_raw(NonNull::dangling().as_ptr()) }); + } + + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. + #[allow(clippy::undocumented_unsafe_blocks)] + let ptr = unsafe { alloc::alloc::alloc_zeroed(layout).cast::() }; + if ptr.is_null() { + return Err(AllocError); + } + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. + #[allow(clippy::undocumented_unsafe_blocks)] + Ok(unsafe { Box::from_raw(ptr) }) + } + + /// Creates a `Box<[Self]>` (a boxed slice) from zeroed bytes. + /// + /// This function is useful for allocating large values of `[Self]` on the + /// heap and zero-initializing them, without ever creating a temporary + /// instance of `[Self; _]` on the stack. For example, + /// `u8::new_box_slice_zeroed(1048576)` will allocate the slice directly on + /// the heap; it does not require storing the slice on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_box_slice_zeroed` may have performance + /// benefits. + /// + /// If `Self` is a zero-sized type, then this function will return a + /// `Box<[Self]>` that has the correct `len`. Such a box cannot contain any + /// actual information, but its `len()` property will report the correct + /// value. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is + /// guaranteed never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_box_zeroed_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline] + fn new_box_zeroed_with_elems(count: usize) -> Result, AllocError> + where + Self: KnownLayout, + { + // SAFETY: `alloc::alloc::alloc_zeroed` is a valid argument of + // `new_box`. The referent of the pointer returned by `alloc_zeroed` + // (and, consequently, the `Box` derived from it) is a valid instance of + // `Self`, because `Self` is `FromZeros`. + unsafe { crate::util::new_box(count, alloc::alloc::alloc_zeroed) } + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromZeros::new_box_zeroed_with_elems`")] + #[doc(hidden)] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[must_use = "has no side effects (other than allocation)"] + #[inline(always)] + fn new_box_slice_zeroed(len: usize) -> Result, AllocError> + where + Self: Sized, + { + <[Self]>::new_box_zeroed_with_elems(len) + } + + /// Creates a `Vec` from zeroed bytes. + /// + /// This function is useful for allocating large values of `Vec`s and + /// zero-initializing them, without ever creating a temporary instance of + /// `[Self; _]` (or many temporary instances of `Self`) on the stack. For + /// example, `u8::new_vec_zeroed(1048576)` will allocate directly on the + /// heap; it does not require storing intermediate values on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_vec_zeroed` may have performance benefits. + /// + /// If `Self` is a zero-sized type, then this function will return a + /// `Vec` that has the correct `len`. Such a `Vec` cannot contain any + /// actual information, but its `len()` property will report the correct + /// value. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is + /// guaranteed never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_vec_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline(always)] + fn new_vec_zeroed(len: usize) -> Result, AllocError> + where + Self: Sized, + { + <[Self]>::new_box_zeroed_with_elems(len).map(Into::into) + } + + /// Extends a `Vec` by pushing `additional` new items onto the end of + /// the vector. The new items are initialized with zeros. + /// + #[doc = codegen_section!( + header = "h5", + bench = "extend_vec_zeroed", + format = "coco_static_size", + )] + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.57.0", feature = "alloc"))))] + #[inline(always)] + fn extend_vec_zeroed(v: &mut Vec, additional: usize) -> Result<(), AllocError> + where + Self: Sized, + { + // PANICS: We pass `v.len()` for `position`, so the `position > v.len()` + // panic condition is not satisfied. + ::insert_vec_zeroed(v, v.len(), additional) + } + + /// Inserts `additional` new items into `Vec` at `position`. The new + /// items are initialized with zeros. + /// + /// # Panics + /// + /// Panics if `position > v.len()`. + /// + #[doc = codegen_section!( + header = "h5", + bench = "insert_vec_zeroed", + format = "coco_static_size", + )] + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.57.0", feature = "alloc"))))] + #[inline] + fn insert_vec_zeroed( + v: &mut Vec, + position: usize, + additional: usize, + ) -> Result<(), AllocError> + where + Self: Sized, + { + assert!(position <= v.len()); + // We only conditionally compile on versions on which `try_reserve` is + // stable; the Clippy lint is a false positive. + v.try_reserve(additional).map_err(|_| AllocError)?; + // SAFETY: The `try_reserve` call guarantees that these cannot overflow: + // * `ptr.add(position)` + // * `position + additional` + // * `v.len() + additional` + // + // `v.len() - position` cannot overflow because we asserted that + // `position <= v.len()`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + // This is a potentially overlapping copy. + let ptr = v.as_mut_ptr(); + #[allow(clippy::arithmetic_side_effects)] + ptr.add(position).copy_to(ptr.add(position + additional), v.len() - position); + ptr.add(position).write_bytes(0, additional); + #[allow(clippy::arithmetic_side_effects)] + v.set_len(v.len() + additional); + } + + Ok(()) + } +} + +/// Analyzes whether a type is [`FromBytes`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `FromBytes` and implements `FromBytes` and its +/// supertraits if it is sound to do so. This derive can be applied to structs, +/// enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromBytes, FromZeros, Immutable}; +/// #[derive(FromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, V01, V02, V03, V04, V05, V06, V07, V08, V09, V0A, V0B, V0C, V0D, V0E, +/// # V0F, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V1A, V1B, V1C, V1D, +/// # V1E, V1F, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V2A, V2B, V2C, +/// # V2D, V2E, V2F, V30, V31, V32, V33, V34, V35, V36, V37, V38, V39, V3A, V3B, +/// # V3C, V3D, V3E, V3F, V40, V41, V42, V43, V44, V45, V46, V47, V48, V49, V4A, +/// # V4B, V4C, V4D, V4E, V4F, V50, V51, V52, V53, V54, V55, V56, V57, V58, V59, +/// # V5A, V5B, V5C, V5D, V5E, V5F, V60, V61, V62, V63, V64, V65, V66, V67, V68, +/// # V69, V6A, V6B, V6C, V6D, V6E, V6F, V70, V71, V72, V73, V74, V75, V76, V77, +/// # V78, V79, V7A, V7B, V7C, V7D, V7E, V7F, V80, V81, V82, V83, V84, V85, V86, +/// # V87, V88, V89, V8A, V8B, V8C, V8D, V8E, V8F, V90, V91, V92, V93, V94, V95, +/// # V96, V97, V98, V99, V9A, V9B, V9C, V9D, V9E, V9F, VA0, VA1, VA2, VA3, VA4, +/// # VA5, VA6, VA7, VA8, VA9, VAA, VAB, VAC, VAD, VAE, VAF, VB0, VB1, VB2, VB3, +/// # VB4, VB5, VB6, VB7, VB8, VB9, VBA, VBB, VBC, VBD, VBE, VBF, VC0, VC1, VC2, +/// # VC3, VC4, VC5, VC6, VC7, VC8, VC9, VCA, VCB, VCC, VCD, VCE, VCF, VD0, VD1, +/// # VD2, VD3, VD4, VD5, VD6, VD7, VD8, VD9, VDA, VDB, VDC, VDD, VDE, VDF, VE0, +/// # VE1, VE2, VE3, VE4, VE5, VE6, VE7, VE8, VE9, VEA, VEB, VEC, VED, VEE, VEF, +/// # VF0, VF1, VF2, VF3, VF4, VF5, VF6, VF7, VF8, VF9, VFA, VFB, VFC, VFD, VFE, +/// # VFF, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@FromBytes#safety +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `FromBytes` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `FromBytes` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `FromBytes` for that type: +/// +/// - If the type is a struct, all of its fields must be `FromBytes`. +/// - If the type is an enum: +/// - It must have a defined representation which is one of `u8`, `u16`, `i8`, +/// or `i16`. +/// - The maximum number of discriminants must be used (so that every possible +/// bit pattern is a valid one). +/// - Its fields must be `FromBytes`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromBytes`, and must *not* rely on the +/// implementation details of this derive. +/// +/// ## Why isn't an explicit representation required for structs? +/// +/// Neither this derive, nor the [safety conditions] of `FromBytes`, requires +/// that structs are marked with `#[repr(C)]`. +/// +/// Per the [Rust reference](reference), +/// +/// > The representation of a type can change the padding between fields, but +/// > does not change the layout of the fields themselves. +/// +/// [reference]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// +/// Since the layout of structs only consists of padding bytes and field bytes, +/// a struct is soundly `FromBytes` if: +/// 1. its padding is soundly `FromBytes`, and +/// 2. its fields are soundly `FromBytes`. +/// +/// The answer to the first question is always yes: padding bytes do not have +/// any validity constraints. A [discussion] of this question in the Unsafe Code +/// Guidelines Working Group concluded that it would be virtually unimaginable +/// for future versions of rustc to add validity constraints to padding bytes. +/// +/// [discussion]: https://github.com/rust-lang/unsafe-code-guidelines/issues/174 +/// +/// Whether a struct is soundly `FromBytes` therefore solely depends on whether +/// its fields are `FromBytes`. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::FromBytes; + +/// Types for which any bit pattern is valid. +/// +/// Any memory region of the appropriate length which contains initialized bytes +/// can be viewed as any `FromBytes` type with no runtime overhead. This is +/// useful for efficiently parsing bytes as structured data. +/// +/// # Warning: Padding bytes +/// +/// Note that, when a value is moved or copied, only the non-padding bytes of +/// that value are guaranteed to be preserved. It is unsound to assume that +/// values written to padding bytes are preserved after a move or copy. For +/// example, the following is unsound: +/// +/// ```rust,no_run +/// use core::mem::{size_of, transmute}; +/// use zerocopy::FromZeros; +/// # use zerocopy_derive::*; +/// +/// // Assume `Foo` is a type with padding bytes. +/// #[derive(FromZeros, Default)] +/// struct Foo { +/// # /* +/// ... +/// # */ +/// } +/// +/// let mut foo: Foo = Foo::default(); +/// FromZeros::zero(&mut foo); +/// // UNSOUND: Although `FromZeros::zero` writes zeros to all bytes of `foo`, +/// // those writes are not guaranteed to be preserved in padding bytes when +/// // `foo` is moved, so this may expose padding bytes as `u8`s. +/// let foo_bytes: [u8; size_of::()] = unsafe { transmute(foo) }; +/// ``` +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(FromBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromBytes, Immutable}; +/// #[derive(FromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, V01, V02, V03, V04, V05, V06, V07, V08, V09, V0A, V0B, V0C, V0D, V0E, +/// # V0F, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V1A, V1B, V1C, V1D, +/// # V1E, V1F, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V2A, V2B, V2C, +/// # V2D, V2E, V2F, V30, V31, V32, V33, V34, V35, V36, V37, V38, V39, V3A, V3B, +/// # V3C, V3D, V3E, V3F, V40, V41, V42, V43, V44, V45, V46, V47, V48, V49, V4A, +/// # V4B, V4C, V4D, V4E, V4F, V50, V51, V52, V53, V54, V55, V56, V57, V58, V59, +/// # V5A, V5B, V5C, V5D, V5E, V5F, V60, V61, V62, V63, V64, V65, V66, V67, V68, +/// # V69, V6A, V6B, V6C, V6D, V6E, V6F, V70, V71, V72, V73, V74, V75, V76, V77, +/// # V78, V79, V7A, V7B, V7C, V7D, V7E, V7F, V80, V81, V82, V83, V84, V85, V86, +/// # V87, V88, V89, V8A, V8B, V8C, V8D, V8E, V8F, V90, V91, V92, V93, V94, V95, +/// # V96, V97, V98, V99, V9A, V9B, V9C, V9D, V9E, V9F, VA0, VA1, VA2, VA3, VA4, +/// # VA5, VA6, VA7, VA8, VA9, VAA, VAB, VAC, VAD, VAE, VAF, VB0, VB1, VB2, VB3, +/// # VB4, VB5, VB6, VB7, VB8, VB9, VBA, VBB, VBC, VBD, VBE, VBF, VC0, VC1, VC2, +/// # VC3, VC4, VC5, VC6, VC7, VC8, VC9, VCA, VCB, VCC, VCD, VCE, VCF, VD0, VD1, +/// # VD2, VD3, VD4, VD5, VD6, VD7, VD8, VD9, VDA, VDB, VDC, VDD, VDE, VDF, VE0, +/// # VE1, VE2, VE3, VE4, VE5, VE6, VE7, VE8, VE9, VEA, VEB, VEC, VED, VEE, VEF, +/// # VF0, VF1, VF2, VF3, VF4, VF5, VF6, VF7, VF8, VF9, VFA, VFB, VFC, VFD, VFE, +/// # VFF, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `FromBytes`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: FromBytes`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `FromBytes` manually, and you don't plan on writing unsafe code that +/// operates on `FromBytes` types, then you don't need to read this section.* +/// +/// If `T: FromBytes`, then unsafe code may assume that it is sound to produce a +/// `T` whose bytes are initialized to any sequence of valid `u8`s (in other +/// words, any byte value which is not uninitialized). If a type is marked as +/// `FromBytes` which violates this contract, it may cause undefined behavior. +/// +/// `#[derive(FromBytes)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::FromBytes", + doc = "[derive-analysis]: zerocopy_derive::FromBytes#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromBytes.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromBytes.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(FromBytes)]` to `{Self}`") +)] +pub unsafe trait FromBytes: FromZeros { + // The `Self: Sized` bound makes it so that `FromBytes` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Interprets the given `source` as a `&Self`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self`. If the length of `source` is not a [valid size of + /// `Self`][valid-size], or if `source` is not appropriately aligned, this + /// returns `Err`. If [`Self: Unaligned`][self-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_bytes(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// header: PacketHeader, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.header.src_port, [0, 1]); + /// assert_eq!(packet.header.dst_port, [2, 3]); + /// assert_eq!(packet.header.length, [4, 5]); + /// assert_eq!(packet.header.checksum, [6, 7]); + /// assert_eq!(packet.body, [8, 9, 10, 11]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_bytes(source: &[u8]) -> Result<&Self, CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_ref(source).try_cast_into_no_leftover::<_, BecauseImmutable>(None) { + Ok(ptr) => Ok(ptr.recall_validity().as_ref()), + Err(err) => Err(err.map_src(|src| src.as_ref())), + } + } + + /// Interprets the prefix of the given `source` as a `&Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`ref_from_prefix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_prefix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// [`ref_from_prefix_with_elems`]: FromBytes::ref_from_prefix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// header: PacketHeader, + /// body: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14][..]; + /// + /// let (packet, suffix) = Packet::ref_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.header.src_port, [0, 1]); + /// assert_eq!(packet.header.dst_port, [2, 3]); + /// assert_eq!(packet.header.length, [4, 5]); + /// assert_eq!(packet.header.checksum, [6, 7]); + /// assert_eq!(packet.body, [[8, 9], [10, 11], [12, 13]]); + /// assert_eq!(suffix, &[14u8][..]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_prefix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_prefix(source: &[u8]) -> Result<(&Self, &[u8]), CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + ref_from_prefix_suffix(source, None, CastType::Prefix) + } + + /// Interprets the suffix of the given bytes as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`ref_from_suffix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_suffix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// [`ref_from_suffix_with_elems`]: FromBytes::ref_from_suffix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::ref_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1, 2, 3, 4, 5][..]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_suffix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_suffix(source: &[u8]) -> Result<(&[u8], &Self), CastError<&[u8], Self>> + where + Self: Immutable + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + ref_from_prefix_suffix(source, None, CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&mut Self`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self`. If the length of `source` is not a [valid size of + /// `Self`][valid-size], or if `source` is not appropriately aligned, this + /// returns `Err`. If [`Self: Unaligned`][self-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`mut_from_prefix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_bytes(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// [`mut_from_prefix_with_elems`]: FromBytes::mut_from_prefix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These bytes encode a `PacketHeader`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let header = PacketHeader::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// + /// header.checksum = [0, 0]; + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 0, 0]); + /// + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_bytes")] + /// + /// See [`FromBytes::ref_from_bytes`](#method.ref_from_bytes.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_bytes(source: &mut [u8]) -> Result<&mut Self, CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_mut(source).try_cast_into_no_leftover::<_, BecauseExclusive>(None) { + Ok(ptr) => Ok(ptr.recall_validity::<_, (_, (_, _))>().as_mut()), + Err(err) => Err(err.map_src(|src| src.as_mut())), + } + } + + /// Interprets the prefix of the given `source` as a `&mut Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`mut_from_suffix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_prefix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// [`mut_from_suffix_with_elems`]: FromBytes::mut_from_suffix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketHeader`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (header, body) = PacketHeader::mut_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// assert_eq!(body, &[8, 9][..]); + /// + /// header.checksum = [0, 0]; + /// body.fill(1); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 0, 0, 1, 1]); + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_prefix")] + /// + /// See [`FromBytes::ref_from_prefix`](#method.ref_from_prefix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_prefix( + source: &mut [u8], + ) -> Result<(&mut Self, &mut [u8]), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + mut_from_prefix_suffix(source, None, CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a `&mut Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_suffix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::mut_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, &[0u8, 1, 2, 3, 4, 5][..]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// + /// prefix.fill(0); + /// trailer.frame_check_sequence.fill(1); + /// + /// assert_eq!(bytes, [0, 0, 0, 0, 0, 0, 1, 1, 1, 1]); + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_suffix")] + /// + /// See [`FromBytes::ref_from_suffix`](#method.ref_from_suffix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_suffix( + source: &mut [u8], + ) -> Result<(&mut [u8], &mut Self), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + mut_from_prefix_suffix(source, None, CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&Self` with a DST length equal to + /// `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, or if `source` is not + /// appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let pixels = <[Pixel]>::ref_from_bytes_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let zsty = ZSTy::ref_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_bytes`]: FromBytes::ref_from_bytes + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_bytes_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_bytes_with_elems( + source: &[u8], + count: usize, + ) -> Result<&Self, CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + let source = Ptr::from_ref(source); + let maybe_slf = source.try_cast_into_no_leftover::<_, BecauseImmutable>(Some(count)); + match maybe_slf { + Ok(slf) => Ok(slf.recall_validity().as_ref()), + Err(err) => Err(err.map_src(|s| s.as_ref())), + } + } + + /// Interprets the prefix of the given `source` as a DST `&Self` with length + /// equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (pixels, suffix) = <[Pixel]>::ref_from_prefix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// assert_eq!(suffix, &[8, 9]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let (zsty, _) = ZSTy::ref_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_prefix`]: FromBytes::ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_prefix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_prefix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&Self, &[u8]), CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + ref_from_prefix_suffix(source, Some(count), CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a DST `&Self` with length + /// equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, pixels) = <[Pixel]>::ref_from_suffix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1]); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 2, g: 3, b: 4, a: 5 }, + /// Pixel { r: 6, g: 7, b: 8, a: 9 }, + /// ]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_suffix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let (_, zsty) = ZSTy::ref_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_suffix`]: FromBytes::ref_from_suffix + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_suffix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_suffix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&[u8], &Self), CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + ref_from_prefix_suffix(source, Some(count), CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&mut Self` with a DST length equal + /// to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, or if `source` is not + /// appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let pixels = <[Pixel]>::mut_from_bytes_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 0, 0, 0, 0]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let zsty = ZSTy::mut_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_bytes`]: FromBytes::mut_from_bytes + /// + #[doc = codegen_header!("h5", "mut_from_bytes_with_elems")] + /// + /// See [`TryFromBytes::ref_from_bytes_with_elems`](#method.ref_from_bytes_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_bytes_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<&mut Self, CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout + Immutable, + { + let source = Ptr::from_mut(source); + let maybe_slf = source.try_cast_into_no_leftover::<_, BecauseImmutable>(Some(count)); + match maybe_slf { + Ok(slf) => Ok(slf.recall_validity::<_, (_, (_, BecauseExclusive))>().as_mut()), + Err(err) => Err(err.map_src(|s| s.as_mut())), + } + } + + /// Interprets the prefix of the given `source` as a `&mut Self` with DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (pixels, suffix) = <[Pixel]>::mut_from_prefix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// assert_eq!(suffix, &[8, 9]); + /// + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// suffix.fill(1); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 0, 0, 0, 0, 1, 1]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let (zsty, _) = ZSTy::mut_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_prefix`]: FromBytes::mut_from_prefix + /// + #[doc = codegen_header!("h5", "mut_from_prefix_with_elems")] + /// + /// See [`TryFromBytes::ref_from_prefix_with_elems`](#method.ref_from_prefix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_prefix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut Self, &mut [u8]), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + mut_from_prefix_suffix(source, Some(count), CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a `&mut Self` with DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, pixels) = <[Pixel]>::mut_from_suffix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1]); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 2, g: 3, b: 4, a: 5 }, + /// Pixel { r: 6, g: 7, b: 8, a: 9 }, + /// ]); + /// + /// prefix.fill(9); + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// + /// assert_eq!(bytes, [9, 9, 2, 3, 4, 5, 0, 0, 0, 0]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_suffix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let (_, zsty) = ZSTy::mut_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_suffix`]: FromBytes::mut_from_suffix + /// + #[doc = codegen_header!("h5", "mut_from_suffix_with_elems")] + /// + /// See [`TryFromBytes::ref_from_suffix_with_elems`](#method.ref_from_suffix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_suffix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut [u8], &mut Self), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + mut_from_prefix_suffix(source, Some(count), CastType::Suffix).map(swap) + } + + /// Reads a copy of `Self` from the given `source`. + /// + /// If `source.len() != size_of::()`, `read_from_bytes` returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These bytes encode a `PacketHeader`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let header = PacketHeader::read_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_bytes", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_bytes(source: &[u8]) -> Result> + where + Self: Sized, + { + match Ref::<_, Unalign>::sized_from(source) { + Ok(r) => Ok(Ref::read(&r).into_inner()), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign` is trivially aligned, so + // `Ref::sized_from` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `Self` from the prefix of the given `source`. + /// + /// This attempts to read a `Self` from the first `size_of::()` bytes + /// of `source`, returning that `Self` and any remaining bytes. If + /// `source.len() < size_of::()`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketHeader`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (header, body) = PacketHeader::read_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// assert_eq!(body, [8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_prefix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_prefix(source: &[u8]) -> Result<(Self, &[u8]), SizeError<&[u8], Self>> + where + Self: Sized, + { + match Ref::<_, Unalign>::sized_from_prefix(source) { + Ok((r, suffix)) => Ok((Ref::read(&r).into_inner(), suffix)), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign` is trivially aligned, so + // `Ref::sized_from_prefix` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `Self` from the suffix of the given `source`. + /// + /// This attempts to read a `Self` from the last `size_of::()` bytes + /// of `source`, returning that `Self` and any preceding bytes. If + /// `source.len() < size_of::()`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::read_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, [0, 1, 2, 3, 4, 5]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_suffix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_suffix(source: &[u8]) -> Result<(&[u8], Self), SizeError<&[u8], Self>> + where + Self: Sized, + { + match Ref::<_, Unalign>::sized_from_suffix(source) { + Ok((prefix, r)) => Ok((prefix, Ref::read(&r).into_inner())), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign` is trivially aligned, so + // `Ref::sized_from_suffix` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `self` from an `io::Read`. + /// + /// This is useful for interfacing with operating system byte sinks (files, + /// sockets, etc.). + /// + /// # Examples + /// + /// ```no_run + /// use zerocopy::{byteorder::big_endian::*, FromBytes}; + /// use std::fs::File; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct BitmapFileHeader { + /// signature: [u8; 2], + /// size: U32, + /// reserved: U64, + /// offset: U64, + /// } + /// + /// let mut file = File::open("image.bin").unwrap(); + /// let header = BitmapFileHeader::read_from_io(&mut file).unwrap(); + /// ``` + #[cfg(feature = "std")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "std")))] + #[inline(always)] + fn read_from_io(mut src: R) -> io::Result + where + Self: Sized, + R: io::Read, + { + // NOTE(#2319, #2320): We do `buf.zero()` separately rather than + // constructing `let buf = CoreMaybeUninit::zeroed()` because, if `Self` + // contains padding bytes, then a typed copy of `CoreMaybeUninit` + // will not necessarily preserve zeros written to those padding byte + // locations, and so `buf` could contain uninitialized bytes. + let mut buf = CoreMaybeUninit::::uninit(); + buf.zero(); + + let ptr = Ptr::from_mut(&mut buf); + // SAFETY: After `buf.zero()`, `buf` consists entirely of initialized, + // zeroed bytes. Since `MaybeUninit` has no validity requirements, `ptr` + // cannot be used to write values which will violate `buf`'s bit + // validity. Since `ptr` has `Exclusive` aliasing, nothing other than + // `ptr` may be used to mutate `ptr`'s referent, and so its bit validity + // cannot be violated even though `buf` may have more permissive bit + // validity than `ptr`. + let ptr = unsafe { ptr.assume_validity::() }; + let ptr = ptr.as_bytes(); + src.read_exact(ptr.as_mut())?; + // SAFETY: `buf` entirely consists of initialized bytes, and `Self` is + // `FromBytes`. + Ok(unsafe { buf.assume_init() }) + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn ref_from(source: &[u8]) -> Option<&Self> + where + Self: KnownLayout + Immutable, + { + Self::ref_from_bytes(source).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_from(source: &mut [u8]) -> Option<&mut Self> + where + Self: KnownLayout + IntoBytes, + { + Self::mut_from_bytes(source).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_prefix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn slice_from_prefix(source: &[u8], count: usize) -> Option<(&[Self], &[u8])> + where + Self: Sized + Immutable, + { + <[Self]>::ref_from_prefix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_suffix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn slice_from_suffix(source: &[u8], count: usize) -> Option<(&[u8], &[Self])> + where + Self: Sized + Immutable, + { + <[Self]>::ref_from_suffix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_prefix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_slice_from_prefix(source: &mut [u8], count: usize) -> Option<(&mut [Self], &mut [u8])> + where + Self: Sized + IntoBytes, + { + <[Self]>::mut_from_prefix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_suffix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_slice_from_suffix(source: &mut [u8], count: usize) -> Option<(&mut [u8], &mut [Self])> + where + Self: Sized + IntoBytes, + { + <[Self]>::mut_from_suffix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::read_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn read_from(source: &[u8]) -> Option + where + Self: Sized, + { + Self::read_from_bytes(source).ok() + } +} + +/// Interprets the given affix of the given bytes as a `&Self`. +/// +/// This method computes the largest possible size of `Self` that can fit in the +/// prefix or suffix bytes of `source`, then attempts to return both a reference +/// to those bytes interpreted as a `Self`, and a reference to the excess bytes. +/// If there are insufficient bytes, or if that affix of `source` is not +/// appropriately aligned, this returns `Err`. +#[inline(always)] +fn ref_from_prefix_suffix( + source: &[u8], + meta: Option, + cast_type: CastType, +) -> Result<(&T, &[u8]), CastError<&[u8], T>> { + let (slf, prefix_suffix) = Ptr::from_ref(source) + .try_cast_into::<_, BecauseImmutable>(cast_type, meta) + .map_err(|err| err.map_src(|s| s.as_ref()))?; + Ok((slf.recall_validity().as_ref(), prefix_suffix.as_ref())) +} + +/// Interprets the given affix of the given bytes as a `&mut Self` without +/// copying. +/// +/// This method computes the largest possible size of `Self` that can fit in the +/// prefix or suffix bytes of `source`, then attempts to return both a reference +/// to those bytes interpreted as a `Self`, and a reference to the excess bytes. +/// If there are insufficient bytes, or if that affix of `source` is not +/// appropriately aligned, this returns `Err`. +#[inline(always)] +fn mut_from_prefix_suffix( + source: &mut [u8], + meta: Option, + cast_type: CastType, +) -> Result<(&mut T, &mut [u8]), CastError<&mut [u8], T>> { + let (slf, prefix_suffix) = Ptr::from_mut(source) + .try_cast_into::<_, BecauseExclusive>(cast_type, meta) + .map_err(|err| err.map_src(|s| s.as_mut()))?; + Ok((slf.recall_validity::<_, (_, (_, _))>().as_mut(), prefix_suffix.as_mut())) +} + +/// Analyzes whether a type is [`IntoBytes`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `IntoBytes` and implements `IntoBytes` if it is +/// sound to do so. This derive can be applied to structs and enums (see below +/// for union support); e.g.: +/// +/// ``` +/// # use zerocopy_derive::{IntoBytes}; +/// #[derive(IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@IntoBytes#safety +/// +/// # Error Messages +/// +/// On Rust toolchains prior to 1.78.0, due to the way that the custom derive +/// for `IntoBytes` is implemented, you may get an error like this: +/// +/// ```text +/// error[E0277]: the trait bound `(): PaddingFree` is not satisfied +/// --> lib.rs:23:10 +/// | +/// 1 | #[derive(IntoBytes)] +/// | ^^^^^^^^^ the trait `PaddingFree` is not implemented for `()` +/// | +/// = help: the following implementations were found: +/// <() as PaddingFree> +/// ``` +/// +/// This error indicates that the type being annotated has padding bytes, which +/// is illegal for `IntoBytes` types. Consider reducing the alignment of some +/// fields by using types in the [`byteorder`] module, wrapping field types in +/// [`Unalign`], adding explicit struct fields where those padding bytes would +/// be, or using `#[repr(packed)]`. See the Rust Reference's page on [type +/// layout] for more information about type layout and padding. +/// +/// [type layout]: https://doc.rust-lang.org/reference/type-layout.html +/// +/// # Unions +/// +/// Currently, union bit validity is [up in the air][union-validity], and so +/// zerocopy does not support `#[derive(IntoBytes)]` on unions by default. +/// However, implementing `IntoBytes` on a union type is likely sound on all +/// existing Rust toolchains - it's just that it may become unsound in the +/// future. You can opt-in to `#[derive(IntoBytes)]` support on unions by +/// passing the unstable `zerocopy_derive_union_into_bytes` cfg: +/// +/// ```shell +/// $ RUSTFLAGS='--cfg zerocopy_derive_union_into_bytes' cargo build +/// ``` +/// +/// However, it is your responsibility to ensure that this derive is sound on +/// the specific versions of the Rust toolchain you are using! We make no +/// stability or soundness guarantees regarding this cfg, and may remove it at +/// any point. +/// +/// We are actively working with Rust to stabilize the necessary language +/// guarantees to support this in a forwards-compatible way, which will enable +/// us to remove the cfg gate. As part of this effort, we need to know how much +/// demand there is for this feature. If you would like to use `IntoBytes` on +/// unions, [please let us know][discussion]. +/// +/// [union-validity]: https://github.com/rust-lang/unsafe-code-guidelines/issues/438 +/// [discussion]: https://github.com/google/zerocopy/discussions/1802 +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `IntoBytes` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `IntoBytes` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `IntoBytes` for that type: +/// +/// - If the type is a struct, its fields must be [`IntoBytes`]. Additionally: +/// - if the type is `repr(transparent)` or `repr(packed)`, it is +/// [`IntoBytes`] if its fields are [`IntoBytes`]; else, +/// - if the type is `repr(C)` with at most one field, it is [`IntoBytes`] +/// if its field is [`IntoBytes`]; else, +/// - if the type has no generic parameters, it is [`IntoBytes`] if the type +/// is sized and has no padding bytes; else, +/// - if the type is `repr(C)`, its fields must be [`Unaligned`]. +/// - If the type is an enum: +/// - It must have a defined representation (`repr`s `C`, `u8`, `u16`, `u32`, +/// `u64`, `usize`, `i8`, `i16`, `i32`, `i64`, or `isize`). +/// - It must have no padding bytes. +/// - Its fields must be [`IntoBytes`]. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromBytes`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [Rust Reference]: https://doc.rust-lang.org/reference/type-layout.html +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::IntoBytes; + +/// Types that can be converted to an immutable slice of initialized bytes. +/// +/// Any `IntoBytes` type can be converted to a slice of initialized bytes of the +/// same size. This is useful for efficiently serializing structured data as raw +/// bytes. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(IntoBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::IntoBytes; +/// #[derive(IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `IntoBytes`. See the [derive +/// documentation][derive] for guidance on how to interpret error messages +/// produced by the derive's analysis. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: IntoBytes`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `IntoBytes` manually, and you don't plan on writing unsafe code that +/// operates on `IntoBytes` types, then you don't need to read this section.* +/// +/// If `T: IntoBytes`, then unsafe code may assume that it is sound to treat any +/// `t: T` as an immutable `[u8]` of length `size_of_val(t)`. If a type is +/// marked as `IntoBytes` which violates this contract, it may cause undefined +/// behavior. +/// +/// `#[derive(IntoBytes)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::IntoBytes", + doc = "[derive-analysis]: zerocopy_derive::IntoBytes#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.IntoBytes.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.IntoBytes.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(IntoBytes)]` to `{Self}`") +)] +pub unsafe trait IntoBytes { + // The `Self: Sized` bound makes it so that this function doesn't prevent + // `IntoBytes` from being object safe. Note that other `IntoBytes` methods + // prevent object safety, but those provide a benefit in exchange for object + // safety. If at some point we remove those methods, change their type + // signatures, or move them out of this trait so that `IntoBytes` is object + // safe again, it's important that this function not prevent object safety. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Gets the bytes of this value. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let bytes = header.as_bytes(); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "as_bytes", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + fn as_bytes(&self) -> &[u8] + where + Self: Immutable, + { + // Note that this method does not have a `Self: Sized` bound; + // `size_of_val` works for unsized values too. + let len = mem::size_of_val(self); + let slf: *const Self = self; + + // SAFETY: + // - `slf.cast::()` is valid for reads for `len * size_of::()` + // many bytes because... + // - `slf` is the same pointer as `self`, and `self` is a reference + // which points to an object whose size is `len`. Thus... + // - The entire region of `len` bytes starting at `slf` is contained + // within a single allocation. + // - `slf` is non-null. + // - `slf` is trivially aligned to `align_of::() == 1`. + // - `Self: IntoBytes` ensures that all of the bytes of `slf` are + // initialized. + // - Since `slf` is derived from `self`, and `self` is an immutable + // reference, the only other references to this memory region that + // could exist are other immutable references, which by `Self: + // Immutable` don't permit mutation. + // - The total size of the resulting slice is no larger than + // `isize::MAX` because no allocation produced by safe code can be + // larger than `isize::MAX`. + // + // FIXME(#429): Add references to docs and quotes. + unsafe { slice::from_raw_parts(slf.cast::(), len) } + } + + /// Gets the bytes of this value mutably. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Eq, PartialEq, Debug)] + /// #[derive(FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let mut header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let bytes = header.as_mut_bytes(); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// + /// bytes.reverse(); + /// + /// assert_eq!(header, PacketHeader { + /// src_port: [7, 6], + /// dst_port: [5, 4], + /// length: [3, 2], + /// checksum: [1, 0], + /// }); + /// ``` + /// + #[doc = codegen_header!("h5", "as_mut_bytes")] + /// + /// See [`IntoBytes::as_bytes`](#method.as_bytes.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + fn as_mut_bytes(&mut self) -> &mut [u8] + where + Self: FromBytes, + { + // Note that this method does not have a `Self: Sized` bound; + // `size_of_val` works for unsized values too. + let len = mem::size_of_val(self); + let slf: *mut Self = self; + + // SAFETY: + // - `slf.cast::()` is valid for reads and writes for `len * + // size_of::()` many bytes because... + // - `slf` is the same pointer as `self`, and `self` is a reference + // which points to an object whose size is `len`. Thus... + // - The entire region of `len` bytes starting at `slf` is contained + // within a single allocation. + // - `slf` is non-null. + // - `slf` is trivially aligned to `align_of::() == 1`. + // - `Self: IntoBytes` ensures that all of the bytes of `slf` are + // initialized. + // - `Self: FromBytes` ensures that no write to this memory region + // could result in it containing an invalid `Self`. + // - Since `slf` is derived from `self`, and `self` is a mutable + // reference, no other references to this memory region can exist. + // - The total size of the resulting slice is no larger than + // `isize::MAX` because no allocation produced by safe code can be + // larger than `isize::MAX`. + // + // FIXME(#429): Add references to docs and quotes. + unsafe { slice::from_raw_parts_mut(slf.cast::(), len) } + } + + /// Writes a copy of `self` to `dst`. + /// + /// If `dst.len() != size_of_val(self)`, `write_to` returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// ``` + /// + /// If too many or too few target bytes are provided, `write_to` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut excessive_bytes = &mut [0u8; 128][..]; + /// + /// let write_result = header.write_to(excessive_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(excessive_bytes, [0u8; 128]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + if dst.len() == src.len() { + // SAFETY: Within this branch of the conditional, we have ensured + // that `dst.len()` is equal to `src.len()`. Neither the size of the + // source nor the size of the destination change between the above + // size check and the invocation of `copy_unchecked`. + unsafe { util::copy_unchecked(src, dst) } + Ok(()) + } else { + Err(SizeError::new(self)) + } + } + + /// Writes a copy of `self` to the prefix of `dst`. + /// + /// `write_to_prefix` writes `self` to the first `size_of_val(self)` bytes + /// of `dst`. If `dst.len() < size_of_val(self)`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to_prefix(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7, 0, 0]); + /// ``` + /// + /// If insufficient target bytes are provided, `write_to_prefix` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to_prefix", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to_prefix(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + match dst.get_mut(..src.len()) { + Some(dst) => { + // SAFETY: Within this branch of the `match`, we have ensured + // through fallible subslicing that `dst.len()` is equal to + // `src.len()`. Neither the size of the source nor the size of + // the destination change between the above subslicing operation + // and the invocation of `copy_unchecked`. + unsafe { util::copy_unchecked(src, dst) } + Ok(()) + } + None => Err(SizeError::new(self)), + } + } + + /// Writes a copy of `self` to the suffix of `dst`. + /// + /// `write_to_suffix` writes `self` to the last `size_of_val(self)` bytes of + /// `dst`. If `dst.len() < size_of_val(self)`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to_suffix(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 0, 0, 1, 2, 3, 4, 5, 6, 7]); + /// + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + /// If insufficient target bytes are provided, `write_to_suffix` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to_suffix", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to_suffix(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + let start = if let Some(start) = dst.len().checked_sub(src.len()) { + start + } else { + return Err(SizeError::new(self)); + }; + let dst = if let Some(dst) = dst.get_mut(start..) { + dst + } else { + // get_mut() should never return None here. We return a `SizeError` + // rather than .unwrap() because in the event the branch is not + // optimized away, returning a value is generally lighter-weight + // than panicking. + return Err(SizeError::new(self)); + }; + // SAFETY: Through fallible subslicing of `dst`, we have ensured that + // `dst.len()` is equal to `src.len()`. Neither the size of the source + // nor the size of the destination change between the above subslicing + // operation and the invocation of `copy_unchecked`. + unsafe { + util::copy_unchecked(src, dst); + } + Ok(()) + } + + /// Writes a copy of `self` to an `io::Write`. + /// + /// This is a shorthand for `dst.write_all(self.as_bytes())`, and is useful + /// for interfacing with operating system byte sinks (files, sockets, etc.). + /// + /// # Examples + /// + /// ```no_run + /// use zerocopy::{byteorder::big_endian::U16, FromBytes, IntoBytes}; + /// use std::fs::File; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct GrayscaleImage { + /// height: U16, + /// width: U16, + /// pixels: [U16], + /// } + /// + /// let image = GrayscaleImage::ref_from_bytes(&[0, 0, 0, 0][..]).unwrap(); + /// let mut file = File::create("image.bin").unwrap(); + /// image.write_to_io(&mut file).unwrap(); + /// ``` + /// + /// If the write fails, `write_to_io` returns `Err` and a partial write may + /// have occurred; e.g.: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// + /// let src = u128::MAX; + /// let mut dst = [0u8; 2]; + /// + /// let write_result = src.write_to_io(&mut dst[..]); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(dst, [255, 255]); + /// ``` + #[cfg(feature = "std")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "std")))] + #[inline(always)] + fn write_to_io(&self, mut dst: W) -> io::Result<()> + where + Self: Immutable, + W: io::Write, + { + dst.write_all(self.as_bytes()) + } + + #[deprecated(since = "0.8.0", note = "`IntoBytes::as_bytes_mut` was renamed to `as_mut_bytes`")] + #[doc(hidden)] + #[inline] + fn as_bytes_mut(&mut self) -> &mut [u8] + where + Self: FromBytes, + { + self.as_mut_bytes() + } +} + +/// Analyzes whether a type is [`Unaligned`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `Unaligned` and implements `Unaligned` if it is +/// sound to do so. This derive can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Unaligned; +/// #[derive(Unaligned)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(packed)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `Unaligned` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `Unaligned` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `Unaligned` for that type: +/// +/// - If the type is a struct or union: +/// - If `repr(align(N))` is provided, `N` must equal 1. +/// - If the type is `repr(C)` or `repr(transparent)`, all fields must be +/// [`Unaligned`]. +/// - If the type is not `repr(C)` or `repr(transparent)`, it must be +/// `repr(packed)` or `repr(packed(1))`. +/// - If the type is an enum: +/// - If `repr(align(N))` is provided, `N` must equal 1. +/// - It must be a field-less enum (meaning that all variants have no fields). +/// - It must be `repr(i8)` or `repr(u8)`. +/// +/// [safety conditions]: trait@Unaligned#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::Unaligned; + +/// Types with no alignment requirement. +/// +/// If `T: Unaligned`, then `align_of::() == 1`. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(Unaligned)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::Unaligned; +/// #[derive(Unaligned)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(packed)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `Unaligned`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: Unaligned`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `Unaligned` manually, and you don't plan on writing unsafe code that +/// operates on `Unaligned` types, then you don't need to read this section.* +/// +/// If `T: Unaligned`, then unsafe code may assume that it is sound to produce a +/// reference to `T` at any memory location regardless of alignment. If a type +/// is marked as `Unaligned` which violates this contract, it may cause +/// undefined behavior. +/// +/// `#[derive(Unaligned)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::Unaligned", + doc = "[derive-analysis]: zerocopy_derive::Unaligned#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Unaligned.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Unaligned.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(Unaligned)]` to `{Self}`") +)] +pub unsafe trait Unaligned { + // The `Self: Sized` bound makes it so that `Unaligned` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; +} + +/// Derives optimized [`PartialEq`] and [`Eq`] implementations. +/// +/// This derive can be applied to structs and enums implementing both +/// [`Immutable`] and [`IntoBytes`]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteEq, Immutable, IntoBytes}; +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The standard library's [`derive(Eq, PartialEq)`][derive@PartialEq] computes +/// equality by individually comparing each field. Instead, the implementation +/// of [`PartialEq::eq`] emitted by `derive(ByteHash)` converts the entirety of +/// `self` and `other` to byte slices and compares those slices for equality. +/// This may have performance advantages. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::ByteEq; +/// Derives an optimized [`Hash`] implementation. +/// +/// This derive can be applied to structs and enums implementing both +/// [`Immutable`] and [`IntoBytes`]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteHash, Immutable, IntoBytes}; +/// #[derive(ByteHash, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(ByteHash, Immutable, IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The standard library's [`derive(Hash)`][derive@Hash] produces hashes by +/// individually hashing each field and combining the results. Instead, the +/// implementations of [`Hash::hash()`] and [`Hash::hash_slice()`] generated by +/// `derive(ByteHash)` convert the entirety of `self` to a byte slice and hashes +/// it in a single call to [`Hasher::write()`]. This may have performance +/// advantages. +/// +/// [`Hash`]: core::hash::Hash +/// [`Hash::hash()`]: core::hash::Hash::hash() +/// [`Hash::hash_slice()`]: core::hash::Hash::hash_slice() +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::ByteHash; +/// Implements [`SplitAt`]. +/// +/// This derive can be applied to structs; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteEq, Immutable, IntoBytes}; +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// ``` +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::SplitAt; + +#[cfg(feature = "alloc")] +#[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] +#[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] +mod alloc_support { + use super::*; + + /// Extends a `Vec` by pushing `additional` new items onto the end of the + /// vector. The new items are initialized with zeros. + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[doc(hidden)] + #[deprecated(since = "0.8.0", note = "moved to `FromZeros`")] + #[inline(always)] + pub fn extend_vec_zeroed( + v: &mut Vec, + additional: usize, + ) -> Result<(), AllocError> { + ::extend_vec_zeroed(v, additional) + } + + /// Inserts `additional` new items into `Vec` at `position`. The new + /// items are initialized with zeros. + /// + /// # Panics + /// + /// Panics if `position > v.len()`. + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[doc(hidden)] + #[deprecated(since = "0.8.0", note = "moved to `FromZeros`")] + #[inline(always)] + pub fn insert_vec_zeroed( + v: &mut Vec, + position: usize, + additional: usize, + ) -> Result<(), AllocError> { + ::insert_vec_zeroed(v, position, additional) + } +} + +#[cfg(feature = "alloc")] +#[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] +#[doc(hidden)] +pub use alloc_support::*; + +#[cfg(test)] +#[allow(clippy::assertions_on_result_states, clippy::unreadable_literal)] +mod tests { + use static_assertions::assert_impl_all; + + use super::*; + use crate::util::testutil::*; + + // An unsized type. + // + // This is used to test the custom derives of our traits. The `[u8]` type + // gets a hand-rolled impl, so it doesn't exercise our custom derives. + #[derive(Debug, Eq, PartialEq, FromBytes, IntoBytes, Unaligned, Immutable)] + #[repr(transparent)] + struct Unsized([u8]); + + impl Unsized { + fn from_mut_slice(slc: &mut [u8]) -> &mut Unsized { + // SAFETY: This *probably* sound - since the layouts of `[u8]` and + // `Unsized` are the same, so are the layouts of `&mut [u8]` and + // `&mut Unsized`. [1] Even if it turns out that this isn't actually + // guaranteed by the language spec, we can just change this since + // it's in test code. + // + // [1] https://github.com/rust-lang/unsafe-code-guidelines/issues/375 + unsafe { mem::transmute(slc) } + } + } + + #[test] + fn test_known_layout() { + // Test that `$ty` and `ManuallyDrop<$ty>` have the expected layout. + // Test that `PhantomData<$ty>` has the same layout as `()` regardless + // of `$ty`. + macro_rules! test { + ($ty:ty, $expect:expr) => { + let expect = $expect; + assert_eq!(<$ty as KnownLayout>::LAYOUT, expect); + assert_eq!( as KnownLayout>::LAYOUT, expect); + assert_eq!( as KnownLayout>::LAYOUT, <() as KnownLayout>::LAYOUT); + }; + } + + let layout = + |offset, align, trailing_slice_elem_size, statically_shallow_unpadded| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: match trailing_slice_elem_size { + None => SizeInfo::Sized { size: offset }, + Some(elem_size) => { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + }, + statically_shallow_unpadded, + }; + + test!((), layout(0, 1, None, false)); + test!(u8, layout(1, 1, None, false)); + // Use `align_of` because `u64` alignment may be smaller than 8 on some + // platforms. + test!(u64, layout(8, mem::align_of::(), None, false)); + test!(AU64, layout(8, 8, None, false)); + + test!(Option<&'static ()>, usize::LAYOUT); + + test!([()], layout(0, 1, Some(0), true)); + test!([u8], layout(0, 1, Some(1), true)); + test!(str, layout(0, 1, Some(1), true)); + } + + #[cfg(feature = "derive")] + #[test] + fn test_known_layout_derive() { + // In this and other files (`late_compile_pass.rs`, + // `mid_compile_pass.rs`, and `struct.rs`), we test success and failure + // modes of `derive(KnownLayout)` for the following combination of + // properties: + // + // +------------+--------------------------------------+-----------+ + // | | trailing field properties | | + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // |------------+----------+----------------+----------+-----------| + // | N | N | N | N | KL00 | + // | N | N | N | Y | KL01 | + // | N | N | Y | N | KL02 | + // | N | N | Y | Y | KL03 | + // | N | Y | N | N | KL04 | + // | N | Y | N | Y | KL05 | + // | N | Y | Y | N | KL06 | + // | N | Y | Y | Y | KL07 | + // | Y | N | N | N | KL08 | + // | Y | N | N | Y | KL09 | + // | Y | N | Y | N | KL10 | + // | Y | N | Y | Y | KL11 | + // | Y | Y | N | N | KL12 | + // | Y | Y | N | Y | KL13 | + // | Y | Y | Y | N | KL14 | + // | Y | Y | Y | Y | KL15 | + // +------------+----------+----------------+----------+-----------+ + + struct NotKnownLayout { + _t: T, + } + + #[derive(KnownLayout)] + #[repr(C)] + struct AlignSize + where + elain::Align: elain::Alignment, + { + _align: elain::Align, + size: [u8; SIZE], + } + + type AU16 = AlignSize<2, 2>; + type AU32 = AlignSize<4, 4>; + + fn _assert_kl(_: &T) {} + + let sized_layout = |align, size| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::Sized { size }, + statically_shallow_unpadded: false, + }; + + let unsized_layout = |align, elem_size, offset, statically_shallow_unpadded| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }), + statically_shallow_unpadded, + }; + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | N | N | Y | KL01 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL01(NotKnownLayout, NotKnownLayout); + + let expected = DstLayout::for_type::(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(4, 8)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(align(64))] + struct KL01Align(NotKnownLayout, NotKnownLayout); + + let expected = DstLayout::for_type::(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(64, 64)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed)] + struct KL01Packed(NotKnownLayout, NotKnownLayout); + + let expected = DstLayout::for_type::(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(1, 6)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed(2))] + struct KL01PackedN(NotKnownLayout, NotKnownLayout); + + assert_impl_all!(KL01PackedN: KnownLayout); + + let expected = DstLayout::for_type::(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(2, 6)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | N | Y | Y | KL03 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL03(NotKnownLayout, u8); + + let expected = DstLayout::for_type::(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(1, 1)); + + // ... with `align(N)` + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(align(64))] + struct KL03Align(NotKnownLayout, u8); + + let expected = DstLayout::for_type::(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(64, 64)); + + // ... with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed)] + struct KL03Packed(NotKnownLayout, u8); + + let expected = DstLayout::for_type::(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(1, 5)); + + // ... with `packed(N)` + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed(2))] + struct KL03PackedN(NotKnownLayout, u8); + + assert_impl_all!(KL03PackedN: KnownLayout); + + let expected = DstLayout::for_type::(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(2, 6)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | Y | N | Y | KL05 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL05(u8, T); + + fn _test_kl05(t: T) -> impl KnownLayout { + KL05(0u8, t) + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | Y | Y | Y | KL07 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL07(u8, T); + + fn _test_kl07(t: T) -> impl KnownLayout { + let _ = KL07(0u8, t); + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | N | Y | N | KL10 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL10(NotKnownLayout, [u8]); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::>(), None) + .extend(<[u8] as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, unsized_layout(4, 1, 4, false)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, align(64))] + struct KL10Align(NotKnownLayout, [u8]); + + let repr_align = NonZeroUsize::new(64); + + let expected = DstLayout::new_zst(repr_align) + .extend(DstLayout::for_type::>(), None) + .extend(<[u8] as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, unsized_layout(64, 1, 4, false)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed)] + struct KL10Packed(NotKnownLayout, [u8]); + + let repr_packed = NonZeroUsize::new(1); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::>(), repr_packed) + .extend(<[u8] as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, unsized_layout(1, 1, 4, false)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed(2))] + struct KL10PackedN(NotKnownLayout, [u8]); + + let repr_packed = NonZeroUsize::new(2); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::>(), repr_packed) + .extend(<[u8] as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, unsized_layout(2, 1, 4, false)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | N | Y | Y | KL11 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL11(NotKnownLayout, u8); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::>(), None) + .extend(::LAYOUT, None) + .pad_to_align(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(8, 16)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, align(64))] + struct KL11Align(NotKnownLayout, u8); + + let repr_align = NonZeroUsize::new(64); + + let expected = DstLayout::new_zst(repr_align) + .extend(DstLayout::for_type::>(), None) + .extend(::LAYOUT, None) + .pad_to_align(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(64, 64)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed)] + struct KL11Packed(NotKnownLayout, u8); + + let repr_packed = NonZeroUsize::new(1); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::>(), repr_packed) + .extend(::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(1, 9)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed(2))] + struct KL11PackedN(NotKnownLayout, u8); + + let repr_packed = NonZeroUsize::new(2); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::>(), repr_packed) + .extend(::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(::LAYOUT, expected); + assert_eq!(::LAYOUT, sized_layout(2, 10)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | Y | Y | N | KL14 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL14(u8, T); + + fn _test_kl14(kl: &KL14) { + _assert_kl(kl) + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | Y | Y | Y | KL15 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL15(u8, T); + + fn _test_kl15(t: T) -> impl KnownLayout { + let _ = KL15(0u8, t); + } + + // Test a variety of combinations of field types: + // - () + // - u8 + // - AU16 + // - [()] + // - [u8] + // - [AU16] + + #[allow(clippy::upper_case_acronyms, dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KLTU(T, U); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(1, 0)); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(1, 1)); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(2, 2)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(1, 0, 0, false)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(1, 1, 0, false)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(2, 2, 0, false)); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(1, 1)); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(1, 2)); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(1, 0, 1, false)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(1, 1, 1, false)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(2, 2, 2, false)); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(2, 2)); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!( as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(2, 0, 2, false)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(2, 1, 2, false)); + + assert_eq!( as KnownLayout>::LAYOUT, unsized_layout(2, 2, 2, false)); + + // Test a variety of field counts. + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF0; + + assert_eq!(::LAYOUT, sized_layout(1, 0)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF1([u8]); + + assert_eq!(::LAYOUT, unsized_layout(1, 1, 0, true)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF2(NotKnownLayout, [u8]); + + assert_eq!(::LAYOUT, unsized_layout(1, 1, 1, false)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF3(NotKnownLayout, NotKnownLayout, [u8]); + + assert_eq!(::LAYOUT, unsized_layout(2, 1, 4, false)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF4(NotKnownLayout, NotKnownLayout, NotKnownLayout, [u8]); + + assert_eq!(::LAYOUT, unsized_layout(4, 1, 8, false)); + } + + #[test] + fn test_object_safety() { + fn _takes_immutable(_: &dyn Immutable) {} + fn _takes_unaligned(_: &dyn Unaligned) {} + } + + #[test] + fn test_from_zeros_only() { + // Test types that implement `FromZeros` but not `FromBytes`. + + assert!(!bool::new_zeroed()); + assert_eq!(char::new_zeroed(), '\0'); + + #[cfg(feature = "alloc")] + { + assert_eq!(bool::new_box_zeroed(), Ok(Box::new(false))); + assert_eq!(char::new_box_zeroed(), Ok(Box::new('\0'))); + + assert_eq!( + <[bool]>::new_box_zeroed_with_elems(3).unwrap().as_ref(), + [false, false, false] + ); + assert_eq!( + <[char]>::new_box_zeroed_with_elems(3).unwrap().as_ref(), + ['\0', '\0', '\0'] + ); + + assert_eq!(bool::new_vec_zeroed(3).unwrap().as_ref(), [false, false, false]); + assert_eq!(char::new_vec_zeroed(3).unwrap().as_ref(), ['\0', '\0', '\0']); + } + + let mut string = "hello".to_string(); + let s: &mut str = string.as_mut(); + assert_eq!(s, "hello"); + s.zero(); + assert_eq!(s, "\0\0\0\0\0"); + } + + #[test] + fn test_zst_count_preserved() { + // Test that, when an explicit count is provided to for a type with a + // ZST trailing slice element, that count is preserved. This is + // important since, for such types, all element counts result in objects + // of the same size, and so the correct behavior is ambiguous. However, + // preserving the count as requested by the user is the behavior that we + // document publicly. + + // FromZeros methods + #[cfg(feature = "alloc")] + assert_eq!(<[()]>::new_box_zeroed_with_elems(3).unwrap().len(), 3); + #[cfg(feature = "alloc")] + assert_eq!(<()>::new_vec_zeroed(3).unwrap().len(), 3); + + // FromBytes methods + assert_eq!(<[()]>::ref_from_bytes_with_elems(&[][..], 3).unwrap().len(), 3); + assert_eq!(<[()]>::ref_from_prefix_with_elems(&[][..], 3).unwrap().0.len(), 3); + assert_eq!(<[()]>::ref_from_suffix_with_elems(&[][..], 3).unwrap().1.len(), 3); + assert_eq!(<[()]>::mut_from_bytes_with_elems(&mut [][..], 3).unwrap().len(), 3); + assert_eq!(<[()]>::mut_from_prefix_with_elems(&mut [][..], 3).unwrap().0.len(), 3); + assert_eq!(<[()]>::mut_from_suffix_with_elems(&mut [][..], 3).unwrap().1.len(), 3); + } + + #[test] + fn test_read_write() { + const VAL: u64 = 0x12345678; + #[cfg(target_endian = "big")] + const VAL_BYTES: [u8; 8] = VAL.to_be_bytes(); + #[cfg(target_endian = "little")] + const VAL_BYTES: [u8; 8] = VAL.to_le_bytes(); + const ZEROS: [u8; 8] = [0u8; 8]; + + // Test `FromBytes::{read_from, read_from_prefix, read_from_suffix}`. + + assert_eq!(u64::read_from_bytes(&VAL_BYTES[..]), Ok(VAL)); + // The first 8 bytes are from `VAL_BYTES` and the second 8 bytes are all + // zeros. + let bytes_with_prefix: [u8; 16] = transmute!([VAL_BYTES, [0; 8]]); + assert_eq!(u64::read_from_prefix(&bytes_with_prefix[..]), Ok((VAL, &ZEROS[..]))); + assert_eq!(u64::read_from_suffix(&bytes_with_prefix[..]), Ok((&VAL_BYTES[..], 0))); + // The first 8 bytes are all zeros and the second 8 bytes are from + // `VAL_BYTES` + let bytes_with_suffix: [u8; 16] = transmute!([[0; 8], VAL_BYTES]); + assert_eq!(u64::read_from_prefix(&bytes_with_suffix[..]), Ok((0, &VAL_BYTES[..]))); + assert_eq!(u64::read_from_suffix(&bytes_with_suffix[..]), Ok((&ZEROS[..], VAL))); + + // Test `IntoBytes::{write_to, write_to_prefix, write_to_suffix}`. + + let mut bytes = [0u8; 8]; + assert_eq!(VAL.write_to(&mut bytes[..]), Ok(())); + assert_eq!(bytes, VAL_BYTES); + let mut bytes = [0u8; 16]; + assert_eq!(VAL.write_to_prefix(&mut bytes[..]), Ok(())); + let want: [u8; 16] = transmute!([VAL_BYTES, [0; 8]]); + assert_eq!(bytes, want); + let mut bytes = [0u8; 16]; + assert_eq!(VAL.write_to_suffix(&mut bytes[..]), Ok(())); + let want: [u8; 16] = transmute!([[0; 8], VAL_BYTES]); + assert_eq!(bytes, want); + } + + #[test] + #[cfg(feature = "std")] + fn test_read_io_with_padding_soundness() { + // This test is designed to exhibit potential UB in + // `FromBytes::read_from_io`. (see #2319, #2320). + + // On most platforms (where `align_of::() == 2`), `WithPadding` + // will have inter-field padding between `x` and `y`. + #[derive(FromBytes)] + #[repr(C)] + struct WithPadding { + x: u8, + y: u16, + } + struct ReadsInRead; + impl std::io::Read for ReadsInRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + // This body branches on every byte of `buf`, ensuring that it + // exhibits UB if any byte of `buf` is uninitialized. + if buf.iter().all(|&x| x == 0) { + Ok(buf.len()) + } else { + buf.iter_mut().for_each(|x| *x = 0); + Ok(buf.len()) + } + } + } + assert!(matches!(WithPadding::read_from_io(ReadsInRead), Ok(WithPadding { x: 0, y: 0 }))); + } + + #[test] + #[cfg(feature = "std")] + fn test_read_write_io() { + let mut long_buffer = [0, 0, 0, 0]; + assert!(matches!(u16::MAX.write_to_io(&mut long_buffer[..]), Ok(()))); + assert_eq!(long_buffer, [255, 255, 0, 0]); + assert!(matches!(u16::read_from_io(&long_buffer[..]), Ok(u16::MAX))); + + let mut short_buffer = [0, 0]; + assert!(u32::MAX.write_to_io(&mut short_buffer[..]).is_err()); + assert_eq!(short_buffer, [255, 255]); + assert!(u32::read_from_io(&short_buffer[..]).is_err()); + } + + #[test] + fn test_try_from_bytes_try_read_from() { + assert_eq!(::try_read_from_bytes(&[0]), Ok(false)); + assert_eq!(::try_read_from_bytes(&[1]), Ok(true)); + + assert_eq!(::try_read_from_prefix(&[0, 2]), Ok((false, &[2][..]))); + assert_eq!(::try_read_from_prefix(&[1, 2]), Ok((true, &[2][..]))); + + assert_eq!(::try_read_from_suffix(&[2, 0]), Ok((&[2][..], false))); + assert_eq!(::try_read_from_suffix(&[2, 1]), Ok((&[2][..], true))); + + // If we don't pass enough bytes, it fails. + assert!(matches!( + ::try_read_from_bytes(&[]), + Err(TryReadError::Size(_)) + )); + assert!(matches!( + ::try_read_from_prefix(&[]), + Err(TryReadError::Size(_)) + )); + assert!(matches!( + ::try_read_from_suffix(&[]), + Err(TryReadError::Size(_)) + )); + + // If we pass too many bytes, it fails. + assert!(matches!( + ::try_read_from_bytes(&[0, 0]), + Err(TryReadError::Size(_)) + )); + + // If we pass an invalid value, it fails. + assert!(matches!( + ::try_read_from_bytes(&[2]), + Err(TryReadError::Validity(_)) + )); + assert!(matches!( + ::try_read_from_prefix(&[2, 0]), + Err(TryReadError::Validity(_)) + )); + assert!(matches!( + ::try_read_from_suffix(&[0, 2]), + Err(TryReadError::Validity(_)) + )); + + // Reading from a misaligned buffer should still succeed. Since `AU64`'s + // alignment is 8, and since we read from two adjacent addresses one + // byte apart, it is guaranteed that at least one of them (though + // possibly both) will be misaligned. + let bytes: [u8; 9] = [0, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!(::try_read_from_bytes(&bytes[..8]), Ok(AU64(0))); + assert_eq!(::try_read_from_bytes(&bytes[1..9]), Ok(AU64(0))); + + assert_eq!( + ::try_read_from_prefix(&bytes[..8]), + Ok((AU64(0), &[][..])) + ); + assert_eq!( + ::try_read_from_prefix(&bytes[1..9]), + Ok((AU64(0), &[][..])) + ); + + assert_eq!( + ::try_read_from_suffix(&bytes[..8]), + Ok((&[][..], AU64(0))) + ); + assert_eq!( + ::try_read_from_suffix(&bytes[1..9]), + Ok((&[][..], AU64(0))) + ); + } + + #[test] + fn test_ref_from_mut_from_bytes() { + // Test `FromBytes::{ref_from_bytes, mut_from_bytes}{,_prefix,Suffix}` + // success cases. Exhaustive coverage for these methods is covered by + // the `Ref` tests above, which these helper methods defer to. + + let mut buf = + Align::<[u8; 16], AU64>::new([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + + assert_eq!( + AU64::ref_from_bytes(&buf.t[8..]).unwrap().0.to_ne_bytes(), + [8, 9, 10, 11, 12, 13, 14, 15] + ); + let suffix = AU64::mut_from_bytes(&mut buf.t[8..]).unwrap(); + suffix.0 = 0x0101010101010101; + // The `[u8:9]` is a non-half size of the full buffer, which would catch + // `from_prefix` having the same implementation as `from_suffix` (issues #506, #511). + assert_eq!( + <[u8; 9]>::ref_from_suffix(&buf.t[..]).unwrap(), + (&[0, 1, 2, 3, 4, 5, 6][..], &[7u8, 1, 1, 1, 1, 1, 1, 1, 1]) + ); + let (prefix, suffix) = AU64::mut_from_suffix(&mut buf.t[1..]).unwrap(); + assert_eq!(prefix, &mut [1u8, 2, 3, 4, 5, 6, 7][..]); + suffix.0 = 0x0202020202020202; + let (prefix, suffix) = <[u8; 10]>::mut_from_suffix(&mut buf.t[..]).unwrap(); + assert_eq!(prefix, &mut [0u8, 1, 2, 3, 4, 5][..]); + suffix[0] = 42; + assert_eq!( + <[u8; 9]>::ref_from_prefix(&buf.t[..]).unwrap(), + (&[0u8, 1, 2, 3, 4, 5, 42, 7, 2], &[2u8, 2, 2, 2, 2, 2, 2][..]) + ); + <[u8; 2]>::mut_from_prefix(&mut buf.t[..]).unwrap().0[1] = 30; + assert_eq!(buf.t, [0, 30, 2, 3, 4, 5, 42, 7, 2, 2, 2, 2, 2, 2, 2, 2]); + } + + #[test] + fn test_ref_from_mut_from_bytes_error() { + // Test `FromBytes::{ref_from_bytes, mut_from_bytes}{,_prefix,Suffix}` + // error cases. + + // Fail because the buffer is too large. + let mut buf = Align::<[u8; 16], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(AU64::ref_from_bytes(&buf.t[..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_bytes(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_bytes(&mut buf.t[..]).is_err()); + + // Fail because the buffer is too small. + let mut buf = Align::<[u8; 4], AU64>::default(); + assert!(AU64::ref_from_bytes(&buf.t[..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_bytes(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(AU64::ref_from_prefix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_prefix(&mut buf.t[..]).is_err()); + assert!(AU64::ref_from_suffix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_suffix(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_prefix(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_prefix(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_suffix(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_suffix(&mut buf.t[..]).is_err()); + + // Fail because the alignment is insufficient. + let mut buf = Align::<[u8; 13], AU64>::default(); + assert!(AU64::ref_from_bytes(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_bytes(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_prefix(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_prefix(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_suffix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_suffix(&mut buf.t[..]).is_err()); + } + + #[test] + fn test_to_methods() { + /// Run a series of tests by calling `IntoBytes` methods on `t`. + /// + /// `bytes` is the expected byte sequence returned from `t.as_bytes()` + /// before `t` has been modified. `post_mutation` is the expected + /// sequence returned from `t.as_bytes()` after `t.as_mut_bytes()[0]` + /// has had its bits flipped (by applying `^= 0xFF`). + /// + /// `N` is the size of `t` in bytes. + fn test( + t: &mut T, + bytes: &[u8], + post_mutation: &T, + ) { + // Test that we can access the underlying bytes, and that we get the + // right bytes and the right number of bytes. + assert_eq!(t.as_bytes(), bytes); + + // Test that changes to the underlying byte slices are reflected in + // the original object. + t.as_mut_bytes()[0] ^= 0xFF; + assert_eq!(t, post_mutation); + t.as_mut_bytes()[0] ^= 0xFF; + + // `write_to` rejects slices that are too small or too large. + assert!(t.write_to(&mut vec![0; N - 1][..]).is_err()); + assert!(t.write_to(&mut vec![0; N + 1][..]).is_err()); + + // `write_to` works as expected. + let mut bytes = [0; N]; + assert_eq!(t.write_to(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_prefix` rejects slices that are too small. + assert!(t.write_to_prefix(&mut vec![0; N - 1][..]).is_err()); + + // `write_to_prefix` works with exact-sized slices. + let mut bytes = [0; N]; + assert_eq!(t.write_to_prefix(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_prefix` works with too-large slices, and any bytes past + // the prefix aren't modified. + let mut too_many_bytes = vec![0; N + 1]; + too_many_bytes[N] = 123; + assert_eq!(t.write_to_prefix(&mut too_many_bytes[..]), Ok(())); + assert_eq!(&too_many_bytes[..N], t.as_bytes()); + assert_eq!(too_many_bytes[N], 123); + + // `write_to_suffix` rejects slices that are too small. + assert!(t.write_to_suffix(&mut vec![0; N - 1][..]).is_err()); + + // `write_to_suffix` works with exact-sized slices. + let mut bytes = [0; N]; + assert_eq!(t.write_to_suffix(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_suffix` works with too-large slices, and any bytes + // before the suffix aren't modified. + let mut too_many_bytes = vec![0; N + 1]; + too_many_bytes[0] = 123; + assert_eq!(t.write_to_suffix(&mut too_many_bytes[..]), Ok(())); + assert_eq!(&too_many_bytes[1..], t.as_bytes()); + assert_eq!(too_many_bytes[0], 123); + } + + #[derive(Debug, Eq, PartialEq, FromBytes, IntoBytes, Immutable)] + #[repr(C)] + struct Foo { + a: u32, + b: Wrapping, + c: Option, + } + + let expected_bytes: Vec = if cfg!(target_endian = "little") { + vec![1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] + } else { + vec![0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0] + }; + let post_mutation_expected_a = + if cfg!(target_endian = "little") { 0x00_00_00_FE } else { 0xFF_00_00_01 }; + test::<_, 12>( + &mut Foo { a: 1, b: Wrapping(2), c: None }, + expected_bytes.as_bytes(), + &Foo { a: post_mutation_expected_a, b: Wrapping(2), c: None }, + ); + test::<_, 3>( + Unsized::from_mut_slice(&mut [1, 2, 3]), + &[1, 2, 3], + Unsized::from_mut_slice(&mut [0xFE, 2, 3]), + ); + } + + #[test] + fn test_array() { + #[derive(FromBytes, IntoBytes, Immutable)] + #[repr(C)] + struct Foo { + a: [u16; 33], + } + + let foo = Foo { a: [0xFFFF; 33] }; + let expected = [0xFFu8; 66]; + assert_eq!(foo.as_bytes(), &expected[..]); + } + + #[test] + fn test_new_zeroed() { + assert!(!bool::new_zeroed()); + assert_eq!(u64::new_zeroed(), 0); + // This test exists in order to exercise unsafe code, especially when + // running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(<()>::new_zeroed(), ()); + } + } + + #[test] + fn test_transparent_packed_generic_struct() { + #[derive(IntoBytes, FromBytes, Unaligned)] + #[repr(transparent)] + #[allow(dead_code)] // We never construct this type + struct Foo { + _t: T, + _phantom: PhantomData<()>, + } + + assert_impl_all!(Foo: FromZeros, FromBytes, IntoBytes); + assert_impl_all!(Foo: Unaligned); + + #[derive(IntoBytes, FromBytes, Unaligned)] + #[repr(C, packed)] + #[allow(dead_code)] // We never construct this type + struct Bar { + _t: T, + _u: U, + } + + assert_impl_all!(Bar: FromZeros, FromBytes, IntoBytes, Unaligned); + } + + #[cfg(feature = "alloc")] + mod alloc { + use super::*; + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_extend_vec_zeroed() { + // Test extending when there is an existing allocation. + let mut v = vec![100u16, 200, 300]; + FromZeros::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 6); + assert_eq!(&*v, &[100, 200, 300, 0, 0, 0]); + drop(v); + + // Test extending when there is no existing allocation. + let mut v: Vec = Vec::new(); + FromZeros::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 3); + assert_eq!(&*v, &[0, 0, 0]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_extend_vec_zeroed_zst() { + // Test extending when there is an existing (fake) allocation. + let mut v = vec![(), (), ()]; + <()>::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 6); + assert_eq!(&*v, &[(), (), (), (), (), ()]); + drop(v); + + // Test extending when there is no existing (fake) allocation. + let mut v: Vec<()> = Vec::new(); + <()>::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(&*v, &[(), (), ()]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_insert_vec_zeroed() { + // Insert at start (no existing allocation). + let mut v: Vec = Vec::new(); + u64::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 2); + assert_eq!(&*v, &[0, 0]); + drop(v); + + // Insert at start. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 5); + assert_eq!(&*v, &[0, 0, 100, 200, 300]); + drop(v); + + // Insert at middle. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 1, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[100, 0, 200, 300]); + drop(v); + + // Insert at end. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 3, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[100, 200, 300, 0]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_insert_vec_zeroed_zst() { + // Insert at start (no existing fake allocation). + let mut v: Vec<()> = Vec::new(); + <()>::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 2); + assert_eq!(&*v, &[(), ()]); + drop(v); + + // Insert at start. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 5); + assert_eq!(&*v, &[(), (), (), (), ()]); + drop(v); + + // Insert at middle. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 1, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[(), (), (), ()]); + drop(v); + + // Insert at end. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 3, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[(), (), (), ()]); + drop(v); + } + + #[test] + fn test_new_box_zeroed() { + assert_eq!(u64::new_box_zeroed(), Ok(Box::new(0))); + } + + #[test] + fn test_new_box_zeroed_array() { + drop(<[u32; 0x1000]>::new_box_zeroed()); + } + + #[test] + fn test_new_box_zeroed_zst() { + // This test exists in order to exercise unsafe code, especially + // when running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(<()>::new_box_zeroed(), Ok(Box::new(()))); + } + } + + #[test] + fn test_new_box_zeroed_with_elems() { + let mut s: Box<[u64]> = <[u64]>::new_box_zeroed_with_elems(3).unwrap(); + assert_eq!(s.len(), 3); + assert_eq!(&*s, &[0, 0, 0]); + s[1] = 3; + assert_eq!(&*s, &[0, 3, 0]); + } + + #[test] + fn test_new_box_zeroed_with_elems_empty() { + let s: Box<[u64]> = <[u64]>::new_box_zeroed_with_elems(0).unwrap(); + assert_eq!(s.len(), 0); + } + + #[test] + fn test_new_box_zeroed_with_elems_zst() { + let mut s: Box<[()]> = <[()]>::new_box_zeroed_with_elems(3).unwrap(); + assert_eq!(s.len(), 3); + assert!(s.get(10).is_none()); + // This test exists in order to exercise unsafe code, especially + // when running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(s[1], ()); + } + s[2] = (); + } + + #[test] + fn test_new_box_zeroed_with_elems_zst_empty() { + let s: Box<[()]> = <[()]>::new_box_zeroed_with_elems(0).unwrap(); + assert_eq!(s.len(), 0); + } + + #[test] + fn new_box_zeroed_with_elems_errors() { + assert_eq!(<[u16]>::new_box_zeroed_with_elems(usize::MAX), Err(AllocError)); + + let max = >::try_from(isize::MAX).unwrap(); + assert_eq!( + <[u16]>::new_box_zeroed_with_elems((max / mem::size_of::()) + 1), + Err(AllocError) + ); + } + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_from_bytes() { + let val = 0u32; + let bytes = val.as_bytes(); + + assert!(u32::ref_from(bytes).is_some()); + // mut_from needs mut bytes + let mut val = 0u32; + let mut_bytes = val.as_mut_bytes(); + assert!(u32::mut_from(mut_bytes).is_some()); + + assert!(u32::read_from(bytes).is_some()); + + let (slc, rest) = ::slice_from_prefix(bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (rest, slc) = ::slice_from_suffix(bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (slc, rest) = ::mut_slice_from_prefix(mut_bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (rest, slc) = ::mut_slice_from_suffix(mut_bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + } + + #[test] + fn test_try_ref_from_prefix_suffix() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 4], u32>::new([0u8; 4]).t[..]; + let (r, rest): (&u32, &[u8]) = u32::try_ref_from_prefix(bytes).unwrap(); + assert_eq!(*r, 0); + assert_eq!(rest.len(), 0); + + let (rest, r): (&[u8], &u32) = u32::try_ref_from_suffix(bytes).unwrap(); + assert_eq!(*r, 0); + assert_eq!(rest.len(), 0); + } + + #[test] + fn test_raw_dangling() { + use crate::util::AsAddress; + let ptr: NonNull = u32::raw_dangling(); + assert_eq!(AsAddress::addr(ptr), 1); + + let ptr: NonNull<[u32]> = <[u32]>::raw_dangling(); + assert_eq!(AsAddress::addr(ptr), 1); + } + + #[test] + fn test_try_ref_from_prefix_with_elems() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 8], u32>::new([0u8; 8]).t[..]; + let (r, rest): (&[u32], &[u8]) = <[u32]>::try_ref_from_prefix_with_elems(bytes, 2).unwrap(); + assert_eq!(r.len(), 2); + assert_eq!(rest.len(), 0); + } + + #[test] + fn test_try_ref_from_suffix_with_elems() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 8], u32>::new([0u8; 8]).t[..]; + let (rest, r): (&[u8], &[u32]) = <[u32]>::try_ref_from_suffix_with_elems(bytes, 2).unwrap(); + assert_eq!(r.len(), 2); + assert_eq!(rest.len(), 0); + } +} diff --git a/rust/zerocopy/src/macros.rs b/rust/zerocopy/src/macros.rs new file mode 100644 index 000000000000..948f8f912bb7 --- /dev/null +++ b/rust/zerocopy/src/macros.rs @@ -0,0 +1,1823 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +/// Safely transmutes a value of one type to a value of another type of the same +/// size. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// const fn transmute(src: Src) -> Dst +/// where +/// Src: IntoBytes, +/// Dst: FromBytes, +/// size_of::() == size_of::(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// However, unlike a function, this macro can only be invoked when the types of +/// `Src` and `Dst` are completely concrete. The types `Src` and `Dst` are +/// inferred from the calling context; they cannot be explicitly specified in +/// the macro invocation. +/// +/// Note that the `Src` produced by the expression `$e` will *not* be dropped. +/// Semantically, its bits will be copied into a new value of type `Dst`, the +/// original `Src` will be forgotten, and the value of type `Dst` will be +/// returned. +/// +/// # `#![allow(shrink)]` +/// +/// If `#![allow(shrink)]` is provided, `transmute!` additionally supports +/// transmutations that shrink the size of the value; e.g.: +/// +/// ``` +/// # use zerocopy::transmute; +/// let u: u32 = transmute!(#![allow(shrink)] 0u64); +/// assert_eq!(u, 0u32); +/// ``` +/// +/// # Examples +/// +/// ``` +/// # use zerocopy::transmute; +/// let one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: [[u8; 4]; 2] = transmute!(one_dimensional); +/// +/// assert_eq!(two_dimensional, [[0, 1, 2, 3], [4, 5, 6, 7]]); +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts. +/// +#[doc = codegen_section!( + header = "h2", + bench = "transmute", + format = "coco_static_size", +)] +#[macro_export] +macro_rules! transmute { + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two types + // have the same size. `core::mem::transmute` uses compiler magic to enforce + // this so long as the types are concrete. + (#![allow(shrink)] $e:expr) => {{ + let mut e = $e; + if false { + // This branch, though never taken, ensures that the type of `e` is + // `IntoBytes` and that the type of the outer macro invocation + // expression is `FromBytes`. + + fn transmute(src: Src) -> Dst + where + Src: $crate::IntoBytes, + Dst: $crate::FromBytes, + { + let _ = src; + loop {} + } + loop {} + #[allow(unreachable_code)] + transmute(e) + } else { + use $crate::util::macro_util::core_reexport::mem::ManuallyDrop; + + // NOTE: `repr(packed)` is important! It ensures that the size of + // `Transmute` won't be rounded up to accommodate `Src`'s or `Dst`'s + // alignment, which would break the size comparison logic below. + // + // As an example of why this is problematic, consider `Src = [u8; + // 5]`, `Dst = u32`. The total size of `Transmute` would + // be 8, and so we would reject a `[u8; 5]` to `u32` transmute as + // being size-increasing, which it isn't. + #[repr(C, packed)] + union Transmute { + src: ManuallyDrop, + dst: ManuallyDrop, + } + + // SAFETY: `Transmute` is a `repr(C)` union whose `src` field has + // type `ManuallyDrop`. Thus, the `src` field starts at byte + // offset 0 within `Transmute` [1]. `ManuallyDrop` has the same + // layout and bit validity as `T`, so it is sound to transmute `Src` + // to `Transmute`. + // + // [1] https://doc.rust-lang.org/1.85.0/reference/type-layout.html#reprc-unions + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop` is guaranteed to have the same layout and bit + // validity as `T` + let u: Transmute<_, _> = unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }; + + if false { + // SAFETY: This code is never executed. + e = ManuallyDrop::into_inner(unsafe { u.src }); + // Suppress the `unused_assignments` lint on the previous line. + let _ = e; + loop {} + } else { + // SAFETY: Per the safety comment on `let u` above, the `dst` + // field in `Transmute` starts at byte offset 0, and has the + // same layout and bit validity as `Dst`. + // + // Transmuting `Src` to `Transmute` above using + // `core::mem::transmute` ensures that `size_of::() == + // size_of::>()`. A `#[repr(C, packed)]` + // union has the maximum size of all of its fields [1], so this + // is equivalent to `size_of::() >= size_of::()`. + // + // The outer `if`'s `false` branch ensures that `Src: IntoBytes` + // and `Dst: FromBytes`. This, combined with the size bound, + // ensures that this transmute is sound. + // + // [1] Per https://doc.rust-lang.org/1.85.0/reference/type-layout.html#reprc-unions: + // + // The union will have a size of the maximum size of all of + // its fields rounded to its alignment + let dst = unsafe { u.dst }; + $crate::util::macro_util::must_use(ManuallyDrop::into_inner(dst)) + } + } + }}; + ($e:expr) => {{ + let e = $e; + if false { + // This branch, though never taken, ensures that the type of `e` is + // `IntoBytes` and that the type of the outer macro invocation + // expression is `FromBytes`. + + fn transmute(src: Src) -> Dst + where + Src: $crate::IntoBytes, + Dst: $crate::FromBytes, + { + let _ = src; + loop {} + } + loop {} + #[allow(unreachable_code)] + transmute(e) + } else { + // SAFETY: `core::mem::transmute` ensures that the type of `e` and + // the type of this macro invocation expression have the same size. + // We know this transmute is safe thanks to the `IntoBytes` and + // `FromBytes` bounds enforced by the `false` branch. + let u = unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations, unnecessary_transmutes)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }; + $crate::util::macro_util::must_use(u) + } + }}; +} + +/// Safely transmutes a mutable or immutable reference of one type to an +/// immutable reference of another type of the same size and compatible +/// alignment. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn transmute_ref<'src, 'dst, Src, Dst>(src: &'src Src) -> &'dst Dst +/// where +/// 'src: 'dst, +/// Src: IntoBytes + Immutable + ?Sized, +/// Dst: FromBytes + Immutable + ?Sized, +/// align_of::() >= align_of::(), +/// size_compatible::(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// # Size compatibility +/// +/// `transmute_ref!` supports transmuting between `Sized` types, between unsized +/// (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. It +/// supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::transmute_ref; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &[[u8; 2]] = &[[0, 1], [2, 3]][..]; +/// let dst: &[u8] = transmute_ref!(src); +/// +/// assert_eq!(src.len(), 2); +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// assert_eq!(size_of_val(src), size_of_val(dst)); +/// ``` +/// +/// # Errors +/// +/// Violations of the alignment and size compatibility checks are detected +/// *after* the compiler performs monomorphization. This has two important +/// consequences. +/// +/// First, it means that generic code will *never* fail these conditions: +/// +/// ``` +/// # use zerocopy::{transmute_ref, FromBytes, IntoBytes, Immutable}; +/// fn transmute_ref(src: &Src) -> &Dst +/// where +/// Src: IntoBytes + Immutable, +/// Dst: FromBytes + Immutable, +/// { +/// transmute_ref!(src) +/// } +/// ``` +/// +/// Instead, failures will only be detected once generic code is instantiated +/// with concrete types: +/// +/// ```compile_fail,E0080 +/// # use zerocopy::{transmute_ref, FromBytes, IntoBytes, Immutable}; +/// # +/// # fn transmute_ref(src: &Src) -> &Dst +/// # where +/// # Src: IntoBytes + Immutable, +/// # Dst: FromBytes + Immutable, +/// # { +/// # transmute_ref!(src) +/// # } +/// let src: &u16 = &0; +/// let dst: &u8 = transmute_ref(src); +/// ``` +/// +/// Second, the fact that violations are detected after monomorphization means +/// that `cargo check` will usually not detect errors, even when types are +/// concrete. Instead, `cargo build` must be used to detect such errors. +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::transmute_ref; +/// let one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: &[[u8; 4]; 2] = transmute_ref!(&one_dimensional); +/// +/// assert_eq!(two_dimensional, &[[0, 1, 2, 3], [4, 5, 6, 7]]); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst; +/// type Dst = SliceDst; +/// +/// let src = Src::ref_from_bytes(&[0, 1, 2, 3, 4, 5, 6, 7]).unwrap(); +/// let dst: &Dst = transmute_ref!(src); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 2, 3]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [4, 5, 6, 7]); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [2, 3, 4, 5, 6, 7]); +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts only when `Src: Sized` and +/// `Dst: Sized`. +/// +#[doc = codegen_section!( + header = "h2", + bench = "transmute_ref", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] +)] +#[macro_export] +macro_rules! transmute_ref { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two + // types have the same size or alignment. + + // Ensure that the source type is a reference or a mutable reference + // (note that mutable references are implicitly reborrowed here). + let e: &_ = $e; + + #[allow(unused, clippy::diverging_sub_expression)] + if false { + // This branch, though never taken, ensures that the type of `e` is + // `&T` where `T: IntoBytes + Immutable`, and that the type of this + // macro expression is `&U` where `U: FromBytes + Immutable`. + + struct AssertSrcIsIntoBytes<'a, T: ?::core::marker::Sized + $crate::IntoBytes>(&'a T); + struct AssertSrcIsImmutable<'a, T: ?::core::marker::Sized + $crate::Immutable>(&'a T); + struct AssertDstIsFromBytes<'a, U: ?::core::marker::Sized + $crate::FromBytes>(&'a U); + struct AssertDstIsImmutable<'a, T: ?::core::marker::Sized + $crate::Immutable>(&'a T); + + let _ = AssertSrcIsIntoBytes(e); + let _ = AssertSrcIsImmutable(e); + + if true { + #[allow(unused, unreachable_code)] + let u = AssertDstIsFromBytes(loop {}); + u.0 + } else { + #[allow(unused, unreachable_code)] + let u = AssertDstIsImmutable(loop {}); + u.0 + } + } else { + use $crate::util::macro_util::TransmuteRefDst; + let t = $crate::util::macro_util::Wrap::new(e); + + if false { + // This branch exists solely to force the compiler to infer the + // type of `Dst` *before* it attempts to resolve the method call + // to `transmute_ref` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `transmute_ref` (which requires `Dst: Sized`) because inherent + // methods take priority over trait methods. It does this before + // it realizes `Dst` is `!Sized`, leading to a compile error when + // it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + t.transmute_ref_inference_helper() + } else { + // SAFETY: The outer `if false` branch ensures that: + // - `Src: IntoBytes + Immutable` + // - `Dst: FromBytes + Immutable` + unsafe { + t.transmute_ref() + } + } + } + }} +} + +/// Safely transmutes a mutable reference of one type to a mutable reference of +/// another type of the same size and compatible alignment. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// const fn transmute_mut<'src, 'dst, Src, Dst>(src: &'src mut Src) -> &'dst mut Dst +/// where +/// 'src: 'dst, +/// Src: FromBytes + IntoBytes + ?Sized, +/// Dst: FromBytes + IntoBytes + ?Sized, +/// align_of::() >= align_of::(), +/// size_compatible::(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// # Size compatibility +/// +/// `transmute_mut!` supports transmuting between `Sized` types, between unsized +/// (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. It +/// supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::transmute_mut; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &mut [[u8; 2]] = &mut [[0, 1], [2, 3]][..]; +/// let dst: &mut [u8] = transmute_mut!(src); +/// +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// let dst_size = size_of_val(dst); +/// assert_eq!(src.len(), 2); +/// assert_eq!(size_of_val(src), dst_size); +/// ``` +/// +/// # Errors +/// +/// Violations of the alignment and size compatibility checks are detected +/// *after* the compiler performs monomorphization. This has two important +/// consequences. +/// +/// First, it means that generic code will *never* fail these conditions: +/// +/// ``` +/// # use zerocopy::{transmute_mut, FromBytes, IntoBytes, Immutable}; +/// fn transmute_mut(src: &mut Src) -> &mut Dst +/// where +/// Src: FromBytes + IntoBytes, +/// Dst: FromBytes + IntoBytes, +/// { +/// transmute_mut!(src) +/// } +/// ``` +/// +/// Instead, failures will only be detected once generic code is instantiated +/// with concrete types: +/// +/// ```compile_fail,E0080 +/// # use zerocopy::{transmute_mut, FromBytes, IntoBytes, Immutable}; +/// # +/// # fn transmute_mut(src: &mut Src) -> &mut Dst +/// # where +/// # Src: FromBytes + IntoBytes, +/// # Dst: FromBytes + IntoBytes, +/// # { +/// # transmute_mut!(src) +/// # } +/// let src: &mut u16 = &mut 0; +/// let dst: &mut u8 = transmute_mut(src); +/// ``` +/// +/// Second, the fact that violations are detected after monomorphization means +/// that `cargo check` will usually not detect errors, even when types are +/// concrete. Instead, `cargo build` must be used to detect such errors. +/// +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::transmute_mut; +/// let mut one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: &mut [[u8; 4]; 2] = transmute_mut!(&mut one_dimensional); +/// +/// assert_eq!(two_dimensional, &[[0, 1, 2, 3], [4, 5, 6, 7]]); +/// +/// two_dimensional.reverse(); +/// +/// assert_eq!(one_dimensional, [4, 5, 6, 7, 0, 1, 2, 3]); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst; +/// type Dst = SliceDst; +/// +/// let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; +/// let src = Src::mut_from_bytes(&mut bytes[..]).unwrap(); +/// let dst: &mut Dst = transmute_mut!(src); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [2, 3, 4, 5, 6, 7]); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 2, 3]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [4, 5, 6, 7]); +/// ``` +#[macro_export] +macro_rules! transmute_mut { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because, for backwards-compatibility on v0.8.x, we use the autoref + // specialization trick to dispatch to different `transmute_mut` + // implementations: one which doesn't require `Src: KnownLayout + Dst: + // KnownLayout` when `Src: Sized + Dst: Sized`, and one which requires + // `KnownLayout` bounds otherwise. + + // Ensure that the source type is a mutable reference. + let e: &mut _ = $e; + + #[allow(unused)] + use $crate::util::macro_util::TransmuteMutDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `transmute_mut` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of `transmute_mut` + // (which requires `Dst: Sized`) because inherent methods take + // priority over trait methods. It does this before it realizes + // `Dst` is `!Sized`, leading to a compile error when it checks the + // bounds later. + // + // By calling this helper (which returns `&mut Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` branch, + // the compiler knows `Dst` is `!Sized`, properly disqualifies the + // inherent method, and falls back to the trait implementation. + t.transmute_mut_inference_helper() + } else { + t.transmute_mut() + } + }} +} + +/// Conditionally transmutes a value of one type to a value of another type of +/// the same size. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute(src: Src) -> Result> +/// where +/// Src: IntoBytes, +/// Dst: TryFromBytes, +/// size_of::() == size_of::(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// However, unlike a function, this macro can only be invoked when the types of +/// `Src` and `Dst` are completely concrete. The types `Src` and `Dst` are +/// inferred from the calling context; they cannot be explicitly specified in +/// the macro invocation. +/// +/// Note that the `Src` produced by the expression `$e` will *not* be dropped. +/// Semantically, its bits will be copied into a new value of type `Dst`, the +/// original `Src` will be forgotten, and the value of type `Dst` will be +/// returned. +/// +/// # Examples +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// assert_eq!(try_transmute!(0u8), Ok(false)); +/// +/// // 1u8 → bool = true +/// assert_eq!(try_transmute!(1u8), Ok(true)); +/// +/// // 2u8 → bool = error +/// assert!(matches!( +/// try_transmute!(2u8), +/// Result::::Err(ValidityError { .. }) +/// )); +/// ``` +/// +#[doc = codegen_section!( + header = "h2", + bench = "try_transmute", + format = "coco_static_size", +)] +#[macro_export] +macro_rules! try_transmute { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two + // types have the same size. `core::mem::transmute` uses compiler magic + // to enforce this so long as the types are concrete. + + let e = $e; + if false { + // Check that the sizes of the source and destination types are + // equal. + + // SAFETY: This code is never executed. + Ok(unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }) + } else { + $crate::util::macro_util::try_transmute::<_, _>(e) + } + }} +} + +/// Conditionally transmutes a mutable or immutable reference of one type to an +/// immutable reference of another type of the same size and compatible +/// alignment. +/// +/// *Note that while the **value** of the referent is checked for validity at +/// runtime, the **size** and **alignment** are checked at compile time. For +/// conversions which are fallible with respect to size and alignment, see the +/// methods on [`TryFromBytes`].* +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute_ref(src: &Src) -> Result<&Dst, ValidityError<&Src, Dst>> +/// where +/// Src: IntoBytes + Immutable + ?Sized, +/// Dst: TryFromBytes + Immutable + ?Sized, +/// align_of::() >= align_of::(), +/// size_compatible::(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// [`TryFromBytes`]: crate::TryFromBytes +/// +/// # Size compatibility +/// +/// `try_transmute_ref!` supports transmuting between `Sized` types, between +/// unsized (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. +/// It supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::try_transmute_ref; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &[[u8; 2]] = &[[0, 1], [2, 3]][..]; +/// let dst: &[u8] = try_transmute_ref!(src).unwrap(); +/// +/// assert_eq!(src.len(), 2); +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// assert_eq!(size_of_val(src), size_of_val(dst)); +/// ``` +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// assert_eq!(try_transmute_ref!(&0u8), Ok(&false)); +/// +/// // 1u8 → bool = true +/// assert_eq!(try_transmute_ref!(&1u8), Ok(&true)); +/// +/// // 2u8 → bool = error +/// assert!(matches!( +/// try_transmute_ref!(&2u8), +/// Result::<&bool, _>::Err(ValidityError { .. }) +/// )); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst; +/// type Dst = SliceDst; +/// +/// let src = Src::ref_from_bytes(&[0, 1, 0, 1, 0, 1, 0, 1]).unwrap(); +/// let dst: &Dst = try_transmute_ref!(src).unwrap(); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 0, 1]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [0, 1, 0, 1]); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [false, true, false, true, false, true]); +/// ``` +/// +#[doc = codegen_section!( + header = "h2", + bench = "try_transmute_ref", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] +)] +#[macro_export] +macro_rules! try_transmute_ref { + ($e:expr) => {{ + // Ensure that the source type is a reference or a mutable reference + // (note that mutable references are implicitly reborrowed here). + let e: &_ = $e; + + #[allow(unused_imports)] + use $crate::util::macro_util::TryTransmuteRefDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `try_transmute_ref` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `try_transmute_ref` (which requires `Dst: Sized`) because + // inherent methods take priority over trait methods. It does this + // before it realizes `Dst` is `!Sized`, leading to a compile error + // when it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + Ok(t.transmute_ref_inference_helper()) + } else { + t.try_transmute_ref() + } + }} +} + +/// Conditionally transmutes a mutable reference of one type to a mutable +/// reference of another type of the same size and compatible alignment. +/// +/// *Note that while the **value** of the referent is checked for validity at +/// runtime, the **size** and **alignment** are checked at compile time. For +/// conversions which are fallible with respect to size and alignment, see the +/// methods on [`TryFromBytes`].* +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute_mut(src: &mut Src) -> Result<&mut Dst, ValidityError<&mut Src, Dst>> +/// where +/// Src: FromBytes + IntoBytes + ?Sized, +/// Dst: TryFromBytes + IntoBytes + ?Sized, +/// align_of::() >= align_of::(), +/// size_compatible::(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// [`TryFromBytes`]: crate::TryFromBytes +/// +/// # Size compatibility +/// +/// `try_transmute_mut!` supports transmuting between `Sized` types, between +/// unsized (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. +/// It supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::try_transmute_mut; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &mut [[u8; 2]] = &mut [[0, 1], [2, 3]][..]; +/// let dst: &mut [u8] = try_transmute_mut!(src).unwrap(); +/// +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// let dst_size = size_of_val(dst); +/// assert_eq!(src.len(), 2); +/// assert_eq!(size_of_val(src), dst_size); +/// ``` +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// let src = &mut 0u8; +/// assert_eq!(try_transmute_mut!(src), Ok(&mut false)); +/// +/// // 1u8 → bool = true +/// let src = &mut 1u8; +/// assert_eq!(try_transmute_mut!(src), Ok(&mut true)); +/// +/// // 2u8 → bool = error +/// let src = &mut 2u8; +/// assert!(matches!( +/// try_transmute_mut!(src), +/// Result::<&mut bool, _>::Err(ValidityError { .. }) +/// )); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst; +/// type Dst = SliceDst; +/// +/// let mut bytes = [0, 1, 0, 1, 0, 1, 0, 1]; +/// let src = Src::mut_from_bytes(&mut bytes).unwrap(); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 0, 1]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [0, 1, 0, 1]); +/// +/// let dst: &Dst = try_transmute_mut!(src).unwrap(); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [false, true, false, true, false, true]); +/// ``` +#[macro_export] +macro_rules! try_transmute_mut { + ($e:expr) => {{ + // Ensure that the source type is a mutable reference. + let e: &mut _ = $e; + + #[allow(unused_imports)] + use $crate::util::macro_util::TryTransmuteMutDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `try_transmute_mut` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `try_transmute_mut` (which requires `Dst: Sized`) because + // inherent methods take priority over trait methods. It does this + // before it realizes `Dst` is `!Sized`, leading to a compile error + // when it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + Ok(t.transmute_mut_inference_helper()) + } else { + t.try_transmute_mut() + } + }} +} + +/// Includes a file and safely transmutes it to a value of an arbitrary type. +/// +/// The file will be included as a byte array, `[u8; N]`, which will be +/// transmuted to another type, `T`. `T` is inferred from the calling context, +/// and must implement [`FromBytes`]. +/// +/// The file is located relative to the current file (similarly to how modules +/// are found). The provided path is interpreted in a platform-specific way at +/// compile time. So, for instance, an invocation with a Windows path containing +/// backslashes `\` would not compile correctly on Unix. +/// +/// `include_value!` is ignorant of byte order. For byte order-aware types, see +/// the [`byteorder`] module. +/// +/// [`FromBytes`]: crate::FromBytes +/// [`byteorder`]: crate::byteorder +/// +/// # Examples +/// +/// Assume there are two files in the same directory with the following +/// contents: +/// +/// File `data` (no trailing newline): +/// +/// ```text +/// abcd +/// ``` +/// +/// File `main.rs`: +/// +/// ```rust +/// use zerocopy::include_value; +/// # macro_rules! include_value { +/// # ($file:expr) => { zerocopy::include_value!(concat!("../testdata/include_value/", $file)) }; +/// # } +/// +/// fn main() { +/// let as_u32: u32 = include_value!("data"); +/// assert_eq!(as_u32, u32::from_ne_bytes([b'a', b'b', b'c', b'd'])); +/// let as_i32: i32 = include_value!("data"); +/// assert_eq!(as_i32, i32::from_ne_bytes([b'a', b'b', b'c', b'd'])); +/// } +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts. +#[doc(alias("include_bytes", "include_data", "include_type"))] +#[macro_export] +macro_rules! include_value { + ($file:expr $(,)?) => { + $crate::transmute!(*::core::include_bytes!($file)) + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! cryptocorrosion_derive_traits { + ( + #[repr($repr:ident)] + $(#[$attr:meta])* + $vis:vis struct $name:ident $(<$($tyvar:ident),*>)? + $( + ( + $($tuple_field_vis:vis $tuple_field_ty:ty),* + ); + )? + + $( + { + $($field_vis:vis $field_name:ident: $field_ty:ty,)* + } + )? + ) => { + $crate::cryptocorrosion_derive_traits!(@assert_allowed_struct_repr #[repr($repr)]); + + $(#[$attr])* + #[repr($repr)] + $vis struct $name $(<$($tyvar),*>)? + $( + ( + $($tuple_field_vis $tuple_field_ty),* + ); + )? + + $( + { + $($field_vis $field_name: $field_ty,)* + } + )? + + // SAFETY: See inline. + unsafe impl $(<$($tyvar),*>)? $crate::TryFromBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + #[inline(always)] + fn is_bit_valid(_: $crate::Maybe<'_, Self, A>) -> bool + where + A: $crate::invariant::Alignment, + { + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block + // requires all field types to be `FromBytes`. Thus, all + // initialized byte sequences constitutes valid instances of + // `Self`. + true + } + + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `FromBytes`, which is a sub-trait of `FromZeros`. + unsafe impl $(<$($tyvar),*>)? $crate::FromZeros for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `FromBytes`. + unsafe impl $(<$($tyvar),*>)? $crate::FromBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, this `impl` block requires all field + // types to be `IntoBytes`, and a padding check is used to ensures that + // there are no padding bytes. + unsafe impl $(<$($tyvar),*>)? $crate::IntoBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::IntoBytes,)* + )? + + $( + $($field_ty: $crate::IntoBytes,)* + )? + + (): $crate::util::macro_util::PaddingFree< + Self, + { + $crate::cryptocorrosion_derive_traits!( + @struct_padding_check #[repr($repr)] + $(($($tuple_field_ty),*))? + $({$($field_ty),*})? + ) + }, + >, + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `Immutable`. + unsafe impl $(<$($tyvar),*>)? $crate::Immutable for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::Immutable,)* + )? + + $( + $($field_ty: $crate::Immutable,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + }; + (@assert_allowed_struct_repr #[repr(transparent)]) => {}; + (@assert_allowed_struct_repr #[repr(C)]) => {}; + (@assert_allowed_struct_repr #[$_attr:meta]) => { + compile_error!("repr must be `#[repr(transparent)]` or `#[repr(C)]`"); + }; + ( + @struct_padding_check #[repr(transparent)] + $(($($tuple_field_ty:ty),*))? + $({$($field_ty:ty),*})? + ) => { + // SAFETY: `#[repr(transparent)]` structs cannot have the same layout as + // their single non-zero-sized field, and so cannot have any padding + // outside of that field. + 0 + }; + ( + @struct_padding_check #[repr(C)] + $(($($tuple_field_ty:ty),*))? + $({$($field_ty:ty),*})? + ) => { + $crate::struct_padding!( + Self, + None, + None, + [ + $($($tuple_field_ty),*)? + $($($field_ty),*)? + ] + ) + }; + ( + #[repr(C)] + $(#[$attr:meta])* + $vis:vis union $name:ident { + $( + $field_name:ident: $field_ty:ty, + )* + } + ) => { + $(#[$attr])* + #[repr(C)] + $vis union $name { + $( + $field_name: $field_ty, + )* + } + + // SAFETY: See inline. + unsafe impl $crate::TryFromBytes for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + #[inline(always)] + fn is_bit_valid(_: $crate::Maybe<'_, Self, A>) -> bool + where + A: $crate::invariant::Alignment, + { + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this + // `impl` block requires all field types to be `FromBytes`. + // Thus, all initialized byte sequences constitutes valid + // instances of `Self`. + true + } + + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `FromBytes`, which is a + // sub-trait of `FromZeros`. + unsafe impl $crate::FromZeros for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `FromBytes`. + unsafe impl $crate::FromBytes for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, this `impl` + // block requires all field types to be `IntoBytes`, and a padding check + // is used to ensures that there are no padding bytes before or after + // any field. + unsafe impl $crate::IntoBytes for $name + where + $( + $field_ty: $crate::IntoBytes, + )* + (): $crate::util::macro_util::PaddingFree< + Self, + { + $crate::union_padding!( + Self, + None::, + None::, + [$($field_ty),*] + ) + }, + >, + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `Immutable`. + unsafe impl $crate::Immutable for $name + where + $( + $field_ty: $crate::Immutable, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + }; +} + +#[cfg(test)] +mod tests { + use crate::{ + byteorder::native_endian::{U16, U32}, + util::testutil::*, + *, + }; + + #[derive(KnownLayout, Immutable, FromBytes, IntoBytes, PartialEq, Debug)] + #[repr(C)] + struct SliceDst { + a: T, + b: [U], + } + + #[test] + fn test_transmute() { + // Test that memory is transmuted as expected. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: [[u8; 2]; 4] = transmute!(array_of_u8s); + assert_eq!(x, array_of_arrays); + let x: [u8; 8] = transmute!(array_of_arrays); + assert_eq!(x, array_of_u8s); + + // Test that memory is transmuted as expected when shrinking. + let x: [[u8; 2]; 3] = transmute!(#![allow(shrink)] array_of_u8s); + assert_eq!(x, [[0u8, 1], [2, 3], [4, 5]]); + + // Test that the source expression's value is forgotten rather than + // dropped. + #[derive(IntoBytes)] + #[repr(transparent)] + struct PanicOnDrop(()); + impl Drop for PanicOnDrop { + fn drop(&mut self) { + panic!("PanicOnDrop::drop"); + } + } + #[allow(clippy::let_unit_value)] + let _: () = transmute!(PanicOnDrop(())); + #[allow(clippy::let_unit_value)] + let _: () = transmute!(#![allow(shrink)] PanicOnDrop(())); + + // Test that `transmute!` is legal in a const context. + const ARRAY_OF_U8S: [u8; 8] = [0u8, 1, 2, 3, 4, 5, 6, 7]; + const ARRAY_OF_ARRAYS: [[u8; 2]; 4] = [[0, 1], [2, 3], [4, 5], [6, 7]]; + const X: [[u8; 2]; 4] = transmute!(ARRAY_OF_U8S); + assert_eq!(X, ARRAY_OF_ARRAYS); + const X_SHRINK: [[u8; 2]; 3] = transmute!(#![allow(shrink)] ARRAY_OF_U8S); + assert_eq!(X_SHRINK, [[0u8, 1], [2, 3], [4, 5]]); + + // Test that `transmute!` works with `!Immutable` types. + let x: usize = transmute!(UnsafeCell::new(1usize)); + assert_eq!(x, 1); + let x: UnsafeCell = transmute!(1usize); + assert_eq!(x.into_inner(), 1); + let x: UnsafeCell = transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.into_inner(), 1); + } + + // A `Sized` type which doesn't implement `KnownLayout` (it is "not + // `KnownLayout`", or `Nkl`). + // + // This permits us to test that `transmute_ref!` and `transmute_mut!` work + // for types which are `Sized + !KnownLayout`. When we added support for + // slice DSTs in #1924, this new support relied on `KnownLayout`, but we + // need to make sure to remain backwards-compatible with code which uses + // these macros with types which are `!KnownLayout`. + #[derive(FromBytes, IntoBytes, Immutable, PartialEq, Eq, Debug)] + #[repr(transparent)] + struct Nkl(T); + + #[test] + fn test_transmute_ref() { + // Test that memory is transmuted as expected. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: &[[u8; 2]; 4] = transmute_ref!(&array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &[u8; 8] = transmute_ref!(&array_of_arrays); + assert_eq!(*x, array_of_u8s); + + // Test that `transmute_ref!` is legal in a const context. + const ARRAY_OF_U8S: [u8; 8] = [0u8, 1, 2, 3, 4, 5, 6, 7]; + const ARRAY_OF_ARRAYS: [[u8; 2]; 4] = [[0, 1], [2, 3], [4, 5], [6, 7]]; + #[allow(clippy::redundant_static_lifetimes)] + const X: &'static [[u8; 2]; 4] = transmute_ref!(&ARRAY_OF_U8S); + assert_eq!(*X, ARRAY_OF_ARRAYS); + + // Test sized -> unsized transmutation. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &array_of_arrays[..]; + let x: &[[u8; 2]] = transmute_ref!(&array_of_u8s); + assert_eq!(x, slice_of_arrays); + + // Before 1.61.0, we can't define the `const fn transmute_ref` function + // that we do on and after 1.61.0. + #[cfg(no_zerocopy_generic_bounds_in_const_fn_1_61_0)] + { + // Test that `transmute_ref!` supports non-`KnownLayout` `Sized` + // types. + const ARRAY_OF_NKL_U8S: Nkl<[u8; 8]> = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + const ARRAY_OF_NKL_ARRAYS: Nkl<[[u8; 2]; 4]> = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + const X_NKL: &Nkl<[[u8; 2]; 4]> = transmute_ref!(&ARRAY_OF_NKL_U8S); + assert_eq!(*X_NKL, ARRAY_OF_NKL_ARRAYS); + } + + #[cfg(not(no_zerocopy_generic_bounds_in_const_fn_1_61_0))] + { + // Call through a generic function to make sure our autoref + // specialization trick works even when types are generic. + const fn transmute_ref(t: &T) -> &U + where + T: IntoBytes + Immutable, + U: FromBytes + Immutable, + { + transmute_ref!(t) + } + + // Test that `transmute_ref!` supports non-`KnownLayout` `Sized` + // types. + const ARRAY_OF_NKL_U8S: Nkl<[u8; 8]> = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + const ARRAY_OF_NKL_ARRAYS: Nkl<[[u8; 2]; 4]> = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + const X_NKL: &Nkl<[[u8; 2]; 4]> = transmute_ref(&ARRAY_OF_NKL_U8S); + assert_eq!(*X_NKL, ARRAY_OF_NKL_ARRAYS); + } + + // Test that `transmute_ref!` works on slice DSTs in and that memory is + // transmuted as expected. + let slice_dst_of_u8s = + SliceDst::::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let slice_dst_of_u16s = + SliceDst::::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: &SliceDst = transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, slice_dst_of_u16s); + + let slice_dst_of_u8s = + SliceDst::::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: &[u8] = transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, [0, 1, 2, 3, 4, 5]); + + let x: &[u8] = transmute_ref!(slice_dst_of_u16s); + assert_eq!(x, [0, 1, 2, 3, 4, 5]); + + let x: &[U16] = transmute_ref!(slice_dst_of_u16s); + let slice_of_u16s: &[U16] = <[U16]>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + assert_eq!(x, slice_of_u16s); + + // Test that transmuting from a type with larger trailing slice offset + // and larger trailing slice element works. + let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + let slice_dst_big = SliceDst::::ref_from_bytes(bytes).unwrap(); + let slice_dst_small = SliceDst::::ref_from_bytes(bytes).unwrap(); + let x: &SliceDst = transmute_ref!(slice_dst_big); + assert_eq!(x, slice_dst_small); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime (note that `X` has the lifetime `'static`). + let x: &[u8; 8] = transmute_ref!(X); + assert_eq!(*x, ARRAY_OF_U8S); + + // Test that `transmute_ref!` supports decreasing alignment. + let u = AU64(0); + let array = [0, 0, 0, 0, 0, 0, 0, 0]; + let x: &[u8; 8] = transmute_ref!(&u); + assert_eq!(*x, array); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: &u8 = transmute_ref!(&mut x); + assert_eq!(*y, 0); + } + + #[test] + fn test_try_transmute() { + // Test that memory is transmuted with `try_transmute` as expected. + let array_of_bools = [false, true, false, true, false, true, false, true]; + let array_of_arrays = [[0, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<[[u8; 2]; 4], _> = try_transmute!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + let x: Result<[bool; 8], _> = try_transmute!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that `try_transmute!` works with `!Immutable` types. + let x: Result = try_transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.unwrap(), 1); + let x: Result, _> = try_transmute!(1usize); + assert_eq!(x.unwrap().into_inner(), 1); + let x: Result, _> = try_transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.unwrap().into_inner(), 1); + + #[derive(FromBytes, IntoBytes, Debug, PartialEq)] + #[repr(transparent)] + struct PanicOnDrop(T); + + impl Drop for PanicOnDrop { + fn drop(&mut self) { + panic!("PanicOnDrop dropped"); + } + } + + // Since `try_transmute!` semantically moves its argument on failure, + // the `PanicOnDrop` is not dropped, and thus this shouldn't panic. + let x: Result = try_transmute!(PanicOnDrop(1usize)); + assert_eq!(x, Ok(1)); + + // Since `try_transmute!` semantically returns ownership of its argument + // on failure, the `PanicOnDrop` is returned rather than dropped, and + // thus this shouldn't panic. + let y: Result = try_transmute!(PanicOnDrop(2u8)); + // We have to use `map_err` instead of comparing against + // `Err(PanicOnDrop(2u8))` because the latter would create and then drop + // its `PanicOnDrop` temporary, which would cause a panic. + assert_eq!(y.as_ref().map_err(|p| &p.src.0), Err::<&bool, _>(&2u8)); + mem::forget(y); + } + + #[test] + fn test_try_transmute_ref() { + // Test that memory is transmuted with `try_transmute_ref` as expected. + let array_of_bools = &[false, true, false, true, false, true, false, true]; + let array_of_arrays = &[[0, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&[[u8; 2]; 4], _> = try_transmute_ref!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + let x: Result<&[bool; 8], _> = try_transmute_ref!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + { + let x: Result<&[[u8; 2]; 4], _> = try_transmute_ref!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + } + + // Test that `try_transmute_ref!` supports decreasing alignment. + let u = AU64(0); + let array = [0u8, 0, 0, 0, 0, 0, 0, 0]; + let x: Result<&[u8; 8], _> = try_transmute_ref!(&u); + assert_eq!(x, Ok(&array)); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: Result<&u8, _> = try_transmute_ref!(&mut x); + assert_eq!(y, Ok(&0)); + + // Test that sized types work which don't implement `KnownLayout`. + let array_of_nkl_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let array_of_nkl_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: Result<&Nkl<[[u8; 2]; 4]>, _> = try_transmute_ref!(&array_of_nkl_u8s); + assert_eq!(x, Ok(&array_of_nkl_arrays)); + + // Test sized -> unsized transmutation. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &array_of_arrays[..]; + let x: Result<&[[u8; 2]], _> = try_transmute_ref!(&array_of_u8s); + assert_eq!(x, Ok(slice_of_arrays)); + + // Test unsized -> unsized transmutation. + let slice_dst_of_u8s = + SliceDst::::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let slice_dst_of_u16s = + SliceDst::::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: Result<&SliceDst, _> = try_transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, Ok(slice_dst_of_u16s)); + } + + #[test] + fn test_try_transmute_mut() { + // Test that memory is transmuted with `try_transmute_mut` as expected. + let array_of_u8s = &mut [0u8, 1, 0, 1, 0, 1, 0, 1]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&mut [[u8; 2]; 4], _> = try_transmute_mut!(array_of_u8s); + assert_eq!(x, Ok(array_of_arrays)); + + let array_of_bools = &mut [false, true, false, true, false, true, false, true]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&mut [bool; 8], _> = try_transmute_mut!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + let array_of_bools = &mut [false, true, false, true, false, true, false, true]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + { + let x: Result<&mut [bool; 8], _> = try_transmute_mut!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + } + + // Test that `try_transmute_mut!` supports decreasing alignment. + let u = &mut AU64(0); + let array = &mut [0u8, 0, 0, 0, 0, 0, 0, 0]; + let x: Result<&mut [u8; 8], _> = try_transmute_mut!(u); + assert_eq!(x, Ok(array)); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: Result<&mut u8, _> = try_transmute_mut!(&mut x); + assert_eq!(y, Ok(&mut 0)); + + // Test that sized types work which don't implement `KnownLayout`. + let mut array_of_nkl_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let mut array_of_nkl_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: Result<&mut Nkl<[[u8; 2]; 4]>, _> = try_transmute_mut!(&mut array_of_nkl_u8s); + assert_eq!(x, Ok(&mut array_of_nkl_arrays)); + + // Test sized -> unsized transmutation. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &mut array_of_arrays[..]; + let x: Result<&mut [[u8; 2]], _> = try_transmute_mut!(&mut array_of_u8s); + assert_eq!(x, Ok(slice_of_arrays)); + + // Test unsized -> unsized transmutation. + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u8s = SliceDst::::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u16s = SliceDst::::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: Result<&mut SliceDst, _> = try_transmute_mut!(slice_dst_of_u8s); + assert_eq!(x, Ok(slice_dst_of_u16s)); + } + + #[test] + fn test_transmute_mut() { + // Test that memory is transmuted as expected. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: &mut [[u8; 2]; 4] = transmute_mut!(&mut array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &mut [u8; 8] = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + + { + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + let x: &mut [u8; 8] = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + } + + // Test that `transmute_mut!` supports non-`KnownLayout` types. + let mut array_of_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let mut array_of_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: &mut Nkl<[[u8; 2]; 4]> = transmute_mut!(&mut array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &mut Nkl<[u8; 8]> = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + + // Test that `transmute_mut!` supports decreasing alignment. + let mut u = AU64(0); + let array = [0, 0, 0, 0, 0, 0, 0, 0]; + let x: &[u8; 8] = transmute_mut!(&mut u); + assert_eq!(*x, array); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: &u8 = transmute_mut!(&mut x); + assert_eq!(*y, 0); + + // Test that `transmute_mut!` works on slice DSTs in and that memory is + // transmuted as expected. + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u8s = SliceDst::::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u16s = SliceDst::::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: &mut SliceDst = transmute_mut!(slice_dst_of_u8s); + assert_eq!(x, slice_dst_of_u16s); + + // Test that `transmute_mut!` works on slices that memory is transmuted + // as expected. + let array_of_u16s: &mut [u16] = &mut [0u16, 1, 2]; + let array_of_i16s: &mut [i16] = &mut [0i16, 1, 2]; + let x: &mut [i16] = transmute_mut!(array_of_u16s); + assert_eq!(x, array_of_i16s); + + // Test that transmuting from a type with larger trailing slice offset + // and larger trailing slice element works. + let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; + let slice_dst_big = SliceDst::::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; + let slice_dst_small = SliceDst::::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: &mut SliceDst = transmute_mut!(slice_dst_big); + assert_eq!(x, slice_dst_small); + + // Test sized -> unsized transmutation. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &mut array_of_arrays[..]; + let x: &mut [[u8; 2]] = transmute_mut!(&mut array_of_u8s); + assert_eq!(x, slice_of_arrays); + } + + #[test] + fn test_macros_evaluate_args_once() { + let mut ctr = 0; + #[allow(clippy::useless_transmute)] + let _: usize = transmute!({ + ctr += 1; + 0usize + }); + assert_eq!(ctr, 1); + + let mut ctr = 0; + let _: &usize = transmute_ref!({ + ctr += 1; + &0usize + }); + assert_eq!(ctr, 1); + + let mut ctr: usize = 0; + let _: &mut usize = transmute_mut!({ + ctr += 1; + &mut ctr + }); + assert_eq!(ctr, 1); + + let mut ctr = 0; + #[allow(clippy::useless_transmute)] + let _: usize = try_transmute!({ + ctr += 1; + 0usize + }) + .unwrap(); + assert_eq!(ctr, 1); + } + + #[test] + fn test_include_value() { + const AS_U32: u32 = include_value!("../testdata/include_value/data"); + assert_eq!(AS_U32, u32::from_ne_bytes([b'a', b'b', b'c', b'd'])); + const AS_I32: i32 = include_value!("../testdata/include_value/data"); + assert_eq!(AS_I32, i32::from_ne_bytes([b'a', b'b', b'c', b'd'])); + } + + #[test] + #[allow(non_camel_case_types, unreachable_pub, dead_code)] + fn test_cryptocorrosion_derive_traits() { + // Test the set of invocations added in + // https://github.com/cryptocorrosion/cryptocorrosion/pull/85 + + fn assert_impls() {} + + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Clone, Copy)] + pub union vec128_storage { + d: [u32; 4], + q: [u64; 2], + } + } + + assert_impls::(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u32x4_generic([u32; 4]); + } + + assert_impls::(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u64x2_generic([u64; 2]); + } + + assert_impls::(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u128x1_generic([u128; 1]); + } + + assert_impls::(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Default)] + #[allow(non_camel_case_types)] + pub struct x2(pub [W; 2], PhantomData); + } + + enum NotZerocopy {} + assert_impls::>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Default)] + #[allow(non_camel_case_types)] + pub struct x4(pub [W; 4]); + } + + assert_impls::>(); + + #[cfg(feature = "simd")] + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + #[cfg(target_arch = "x86")] + use core::arch::x86::{__m128i, __m256i}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{__m128i, __m256i}; + + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Copy, Clone)] + pub struct X4(__m128i, __m128i, __m128i, __m128i); + } + + assert_impls::(); + + cryptocorrosion_derive_traits! { + #[repr(C)] + /// Generic wrapper for unparameterized storage of any of the + /// possible impls. Converting into and out of this type should + /// be essentially free, although it may be more aligned than a + /// particular impl requires. + #[allow(non_camel_case_types)] + #[derive(Copy, Clone)] + pub union vec128_storage { + u32x4: [u32; 4], + u64x2: [u64; 2], + u128x1: [u128; 1], + sse2: __m128i, + } + } + + assert_impls::(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[allow(non_camel_case_types)] + #[derive(Copy, Clone)] + pub struct vec { + x: __m128i, + s3: PhantomData, + s4: PhantomData, + ni: PhantomData, + } + } + + assert_impls::>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone)] + pub struct u32x4x2_avx2 { + x: __m256i, + ni: PhantomData, + } + } + + assert_impls::>(); + } + + // Make sure that our derive works for `#[repr(C)]` structs even though + // cryptocorrosion doesn't currently have any. + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct ReprC(u8, u8, u16); + } + } +} diff --git a/rust/zerocopy/src/pointer/inner.rs b/rust/zerocopy/src/pointer/inner.rs new file mode 100644 index 000000000000..c8dd9f01f456 --- /dev/null +++ b/rust/zerocopy/src/pointer/inner.rs @@ -0,0 +1,752 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{marker::PhantomData, ops::Range, ptr::NonNull}; + +pub use _def::PtrInner; + +#[allow(unused_imports)] +use crate::util::polyfills::NumExt as _; +use crate::{ + layout::{CastType, MetadataCastError}, + pointer::cast, + util::AsAddress, + AlignmentError, CastError, KnownLayout, MetadataOf, SizeError, SplitAt, +}; + +mod _def { + use super::*; + /// The inner pointer stored inside a [`Ptr`][crate::Ptr]. + /// + /// `PtrInner<'a, T>` is [covariant] in `'a` and invariant in `T`. + /// + /// [covariant]: https://doc.rust-lang.org/reference/subtyping.html + #[allow(missing_debug_implementations)] + pub struct PtrInner<'a, T> + where + T: ?Sized, + { + /// # Invariants + /// + /// 0. If `ptr`'s referent is not zero sized, then `ptr` has valid + /// provenance for its referent, which is entirely contained in some + /// Rust allocation, `A`. + /// 1. If `ptr`'s referent is not zero sized, `A` is guaranteed to live + /// for at least `'a`. + /// + /// # Postconditions + /// + /// By virtue of these invariants, code may assume the following, which + /// are logical implications of the invariants: + /// - `ptr`'s referent is not larger than `isize::MAX` bytes \[1\] + /// - `ptr`'s referent does not wrap around the address space \[1\] + /// + /// \[1\] Per : + /// + /// For any allocated object with `base` address, `size`, and a set of + /// `addresses`, the following are guaranteed: + /// ... + /// - `size <= isize::MAX` + /// + /// As a consequence of these guarantees, given any address `a` within + /// the set of addresses of an allocated object: + /// ... + /// - It is guaranteed that, given `o = a - base` (i.e., the offset of + /// `a` within the allocated object), `base + o` will not wrap + /// around the address space (in other words, will not overflow + /// `usize`) + ptr: NonNull, + // SAFETY: `&'a UnsafeCell` is covariant in `'a` and invariant in `T` + // [1]. We use this construction rather than the equivalent `&mut T`, + // because our MSRV of 1.65 prohibits `&mut` types in const contexts. + // + // [1] https://doc.rust-lang.org/1.81.0/reference/subtyping.html#variance + _marker: PhantomData<&'a core::cell::UnsafeCell>, + } + + impl<'a, T: 'a + ?Sized> Copy for PtrInner<'a, T> {} + impl<'a, T: 'a + ?Sized> Clone for PtrInner<'a, T> { + #[inline(always)] + fn clone(&self) -> PtrInner<'a, T> { + // SAFETY: None of the invariants on `ptr` are affected by having + // multiple copies of a `PtrInner`. + *self + } + } + + impl<'a, T: 'a + ?Sized> PtrInner<'a, T> { + /// Constructs a `Ptr` from a [`NonNull`]. + /// + /// # Safety + /// + /// The caller promises that: + /// + /// 0. If `ptr`'s referent is not zero sized, then `ptr` has valid + /// provenance for its referent, which is entirely contained in some + /// Rust allocation, `A`. + /// 1. If `ptr`'s referent is not zero sized, `A` is guaranteed to live + /// for at least `'a`. + #[inline(always)] + #[must_use] + pub const unsafe fn new(ptr: NonNull) -> PtrInner<'a, T> { + // SAFETY: The caller has promised to satisfy all safety invariants + // of `PtrInner`. + Self { ptr, _marker: PhantomData } + } + + /// Converts this `PtrInner` to a [`NonNull`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned `NonNull` in a + /// way that violates the safety invariants of `self`. + #[inline(always)] + #[must_use] + pub const fn as_non_null(&self) -> NonNull { + self.ptr + } + + /// Converts this `PtrInner` to a [`*mut T`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned `*mut T` in a + /// way that violates the safety invariants of `self`. + #[inline(always)] + #[must_use] + pub const fn as_ptr(&self) -> *mut T { + self.ptr.as_ptr() + } + } +} + +impl<'a, T: ?Sized> PtrInner<'a, T> { + /// Constructs a `PtrInner` from a reference. + #[inline] + pub fn from_ref(ptr: &'a T) -> Self { + let ptr = NonNull::from(ptr); + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr`, by invariant on + // `&'a T` [1], has valid provenance for its referent, which is + // entirely contained in some Rust allocation, `A`. + // 1. If `ptr`'s referent is not zero sized, then `A`, by invariant on + // `&'a T`, is guaranteed to live for at least `'a`. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: &mut T`, + // when such values cross an API boundary, the following invariants + // must generally be upheld: + // ... + // - if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being “dereferenceable” for N bytes + // means that the memory range `[a, a + N)` is all contained within a + // single allocated object. + unsafe { Self::new(ptr) } + } + + /// Constructs a `PtrInner` from a mutable reference. + #[inline] + pub fn from_mut(ptr: &'a mut T) -> Self { + let ptr = NonNull::from(ptr); + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr`, by invariant on + // `&'a mut T` [1], has valid provenance for its referent, which is + // entirely contained in some Rust allocation, `A`. + // 1. If `ptr`'s referent is not zero sized, then `A`, by invariant on + // `&'a mut T`, is guaranteed to live for at least `'a`. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: &mut T`, + // when such values cross an API boundary, the following invariants + // must generally be upheld: + // ... + // - if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being “dereferenceable” for N bytes + // means that the memory range `[a, a + N)` is all contained within a + // single allocated object. + unsafe { Self::new(ptr) } + } + + /// # Safety + /// + /// The caller may assume that the resulting `PtrInner` addresses the subset + /// of the bytes of `self`'s referent addressed by `C::project(self)`. + #[must_use] + #[inline(always)] + pub fn project>(self) -> PtrInner<'a, U> { + let projected_raw = C::project(self); + + // SAFETY: `self`'s referent lives at a `NonNull` address, and is either + // zero-sized or lives in an allocation. In either case, it does not + // wrap around the address space [1], and so none of the addresses + // contained in it or one-past-the-end of it are null. + // + // By invariant on `C: Project`, `C::project` is a provenance-preserving + // projection which preserves or shrinks the set of referent bytes, so + // `projected_raw` references a subset of `self`'s referent, and so it + // cannot be null. + // + // [1] https://doc.rust-lang.org/1.92.0/std/ptr/index.html#allocation + let projected_non_null = unsafe { NonNull::new_unchecked(projected_raw) }; + + // SAFETY: As described in the preceding safety comment, `projected_raw`, + // and thus `projected_non_null`, addresses a subset of `self`'s + // referent. Thus, `projected_non_null` either: + // - Addresses zero bytes or, + // - Addresses a subset of the referent of `self`. In this case, `self` + // has provenance for its referent, which lives in an allocation. + // Since `projected_non_null` was constructed using a sequence of + // provenance-preserving operations, it also has provenance for its + // referent and that referent lives in an allocation. By invariant on + // `self`, that allocation lives for `'a`. + unsafe { PtrInner::new(projected_non_null) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, T> +where + T: ?Sized + KnownLayout, +{ + /// Extracts the metadata of this `ptr`. + #[inline] + #[must_use] + pub fn meta(self) -> MetadataOf { + let meta = T::pointer_to_metadata(self.as_ptr()); + // SAFETY: By invariant on `PtrInner`, `self.as_non_null()` addresses no + // more than `isize::MAX` bytes. + unsafe { MetadataOf::new_unchecked(meta) } + } + + /// Produces a `PtrInner` with the same address and provenance as `self` but + /// the given `meta`. + /// + /// # Safety + /// + /// The caller promises that if `self`'s referent is not zero sized, then + /// a pointer constructed from its address with the given `meta` metadata + /// will address a subset of the allocation pointed to by `self`. + #[inline] + #[must_use] + pub unsafe fn with_meta(self, meta: T::PointerMetadata) -> Self + where + T: KnownLayout, + { + let raw = T::raw_from_ptr_len(self.as_non_null().cast(), meta); + + // SAFETY: + // + // Lemma 0: `raw` either addresses zero bytes, or addresses a subset of + // the allocation pointed to by `self` and has the same + // provenance as `self`. Proof: `raw` is constructed using + // provenance-preserving operations, and the caller has + // promised that, if `self`'s referent is not zero-sized, the + // resulting pointer addresses a subset of the allocation + // pointed to by `self`. + // + // 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` is derived from some valid Rust allocation, + // `A`. + // 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` has valid provenance for `A`. + // 2. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` addresses a byte range which is entirely + // contained in `A`. + // 3. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte + // range whose length fits in an `isize`. + // 4. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte + // range which does not wrap around the address space. + // 5. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `A` is guaranteed to live for at least `'a`. + unsafe { PtrInner::new(raw) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, T> +where + T: ?Sized + KnownLayout, +{ + /// Splits `T` in two. + /// + /// # Safety + /// + /// The caller promises that: + /// - `l_len.get() <= self.meta()`. + /// + /// ## (Non-)Overlap + /// + /// Given `let (left, right) = ptr.split_at(l_len)`, it is guaranteed that + /// `left` and `right` are contiguous and non-overlapping if + /// `l_len.padding_needed_for() == 0`. This is true for all `[T]`. + /// + /// If `l_len.padding_needed_for() != 0`, then the left pointer will overlap + /// the right pointer to satisfy `T`'s padding requirements. + #[inline] + #[must_use] + pub unsafe fn split_at_unchecked( + self, + l_len: crate::util::MetadataOf, + ) -> (Self, PtrInner<'a, [T::Elem]>) + where + T: SplitAt, + { + let l_len = l_len.get(); + + // SAFETY: The caller promises that `l_len.get() <= self.meta()`. + // Trivially, `0 <= l_len`. + let left = unsafe { self.with_meta(l_len) }; + + let right = self.trailing_slice(); + // SAFETY: The caller promises that `l_len <= self.meta() = slf.meta()`. + // Trivially, `slf.meta() <= slf.meta()`. + let right = unsafe { right.slice_unchecked(l_len..self.meta().get()) }; + + // SAFETY: If `l_len.padding_needed_for() == 0`, then `left` and `right` + // are non-overlapping. Proof: `left` is constructed `slf` with `l_len` + // as its (exclusive) upper bound. If `l_len.padding_needed_for() == 0`, + // then `left` requires no trailing padding following its final element. + // Since `right` is constructed from `slf`'s trailing slice with `l_len` + // as its (inclusive) lower bound, no byte is referred to by both + // pointers. + // + // Conversely, `l_len.padding_needed_for() == N`, where `N + // > 0`, `left` requires `N` bytes of trailing padding following its + // final element. Since `right` is constructed from the trailing slice + // of `slf` with `l_len` as its (inclusive) lower bound, the first `N` + // bytes of `right` are aliased by `left`. + (left, right) + } + + /// Produces the trailing slice of `self`. + #[inline] + #[must_use] + pub fn trailing_slice(self) -> PtrInner<'a, [T::Elem]> + where + T: SplitAt, + { + let offset = crate::trailing_slice_layout::().offset; + + let bytes = self.as_non_null().cast::().as_ptr(); + + // SAFETY: + // - By invariant on `T: KnownLayout`, `T::LAYOUT` describes `T`'s + // layout. `offset` is the offset of the trailing slice within `T`, + // which is by definition in-bounds or one byte past the end of any + // `T`, regardless of metadata. By invariant on `PtrInner`, `self` + // (and thus `bytes`) points to a byte range of size `<= isize::MAX`, + // and so `offset <= isize::MAX`. Since `size_of::() == 1`, + // `offset * size_of::() <= isize::MAX`. + // - If `offset > 0`, then by invariant on `PtrInner`, `self` (and thus + // `bytes`) points to a byte range entirely contained within the same + // allocated object as `self`. As explained above, this offset results + // in a pointer to or one byte past the end of this allocated object. + let bytes = unsafe { bytes.add(offset) }; + + // SAFETY: By the preceding safety argument, `bytes` is within or one + // byte past the end of the same allocated object as `self`, which + // ensures that it is non-null. + let bytes = unsafe { NonNull::new_unchecked(bytes) }; + + let ptr = KnownLayout::raw_from_ptr_len(bytes, self.meta().get()); + + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr` is derived from + // some valid Rust allocation, `A`, because `ptr` is derived from + // the same allocated object as `self`. + // 1. If `ptr`'s referent is not zero sized, then `ptr` has valid + // provenance for `A` because `raw` is derived from the same + // allocated object as `self` via provenance-preserving operations. + // 2. If `ptr`'s referent is not zero sized, then `ptr` addresses a byte + // range which is entirely contained in `A`, by previous safety proof + // on `bytes`. + // 3. `ptr` addresses a byte range whose length fits in an `isize`, by + // consequence of #2. + // 4. `ptr` addresses a byte range which does not wrap around the + // address space, by consequence of #2. + // 5. If `ptr`'s referent is not zero sized, then `A` is guaranteed to + // live for at least `'a`, because `ptr` is derived from `self`. + unsafe { PtrInner::new(ptr) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, [T]> { + /// Creates a pointer which addresses the given `range` of self. + /// + /// # Safety + /// + /// `range` is a valid range (`start <= end`) and `end <= self.meta()`. + #[inline] + #[must_use] + pub unsafe fn slice_unchecked(self, range: Range) -> Self { + let base = self.as_non_null().cast::().as_ptr(); + + // SAFETY: The caller promises that `start <= end <= self.meta()`. By + // invariant, if `self`'s referent is not zero-sized, then `self` refers + // to a byte range which is contained within a single allocation, which + // is no more than `isize::MAX` bytes long, and which does not wrap + // around the address space. Thus, this pointer arithmetic remains + // in-bounds of the same allocation, and does not wrap around the + // address space. The offset (in bytes) does not overflow `isize`. + // + // If `self`'s referent is zero-sized, then these conditions are + // trivially satisfied. + let base = unsafe { base.add(range.start) }; + + // SAFETY: The caller promises that `start <= end`, and so this will not + // underflow. + #[allow(unstable_name_collisions)] + let len = unsafe { range.end.unchecked_sub(range.start) }; + + let ptr = core::ptr::slice_from_raw_parts_mut(base, len); + + // SAFETY: By invariant, `self`'s referent is either a ZST or lives + // entirely in an allocation. `ptr` points inside of or one byte past + // the end of that referent. Thus, in either case, `ptr` is non-null. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: + // + // Lemma 0: `ptr` addresses a subset of the bytes addressed by `self`, + // and has the same provenance. Proof: The caller guarantees + // that `start <= end <= self.meta()`. Thus, `base` is + // in-bounds of `self`, and `base + (end - start)` is also + // in-bounds of self. Finally, `ptr` is constructed using + // provenance-preserving operations. + // + // 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` has valid provenance for its referent, + // which is entirely contained in some Rust allocation, `A`. + // 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `A` is guaranteed to live for at least `'a`. + unsafe { PtrInner::new(ptr) } + } + + /// Iteratively projects the elements `PtrInner` from `PtrInner<[T]>`. + #[inline] + pub fn iter(&self) -> impl Iterator> { + // FIXME(#429): Once `NonNull::cast` documents that it preserves + // provenance, cite those docs. + let base = self.as_non_null().cast::().as_ptr(); + (0..self.meta().get()).map(move |i| { + // FIXME(https://github.com/rust-lang/rust/issues/74265): Use + // `NonNull::get_unchecked_mut`. + + // SAFETY: If the following conditions are not satisfied + // `pointer::cast` may induce Undefined Behavior [1]: + // + // > - The computed offset, `count * size_of::()` bytes, must not + // > overflow `isize``. + // > - If the computed offset is non-zero, then `self` must be + // > derived from a pointer to some allocated object, and the + // > entire memory range between `self` and the result must be in + // > bounds of that allocated object. In particular, this range + // > must not “wrap around” the edge of the address space. + // + // [1] https://doc.rust-lang.org/std/primitive.pointer.html#method.add + // + // We satisfy both of these conditions here: + // - By invariant on `Ptr`, `self` addresses a byte range whose + // length fits in an `isize`. Since `elem` is contained in `self`, + // the computed offset of `elem` must fit within `isize.` + // - If the computed offset is non-zero, then this means that the + // referent is not zero-sized. In this case, `base` points to an + // allocated object (by invariant on `self`). Thus: + // - By contract, `self.meta()` accurately reflects the number of + // elements in the slice. `i` is in bounds of `c.meta()` by + // construction, and so the result of this addition cannot + // overflow past the end of the allocation referred to by `c`. + // - By invariant on `Ptr`, `self` addresses a byte range which + // does not wrap around the address space. Since `elem` is + // contained in `self`, the computed offset of `elem` must wrap + // around the address space. + // + // FIXME(#429): Once `pointer::add` documents that it preserves + // provenance, cite those docs. + let elem = unsafe { base.add(i) }; + + // SAFETY: `elem` must not be null. `base` is constructed from a + // `NonNull` pointer, and the addition that produces `elem` must not + // overflow or wrap around, so `elem >= base > 0`. + // + // FIXME(#429): Once `NonNull::new_unchecked` documents that it + // preserves provenance, cite those docs. + let elem = unsafe { NonNull::new_unchecked(elem) }; + + // SAFETY: The safety invariants of `Ptr::new` (see definition) are + // satisfied: + // 0. If `elem`'s referent is not zero sized, then `elem` has valid + // provenance for its referent, because it derived from `self` + // using a series of provenance-preserving operations, and + // because `self` has valid provenance for its referent. By the + // same argument, `elem`'s referent is entirely contained within + // the same allocated object as `self`'s referent. + // 1. If `elem`'s referent is not zero sized, then the allocation of + // `elem` is guaranteed to live for at least `'a`, because `elem` + // is entirely contained in `self`, which lives for at least `'a` + // by invariant on `Ptr`. + unsafe { PtrInner::new(elem) } + }) + } +} + +impl<'a, T, const N: usize> PtrInner<'a, [T; N]> { + /// Casts this pointer-to-array into a slice. + /// + /// # Safety + /// + /// Callers may assume that the returned `PtrInner` references the same + /// address and length as `self`. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_slice(self) -> PtrInner<'a, [T]> { + let start = self.as_non_null().cast::().as_ptr(); + let slice = core::ptr::slice_from_raw_parts_mut(start, N); + // SAFETY: `slice` is not null, because it is derived from `start` + // which is non-null. + let slice = unsafe { NonNull::new_unchecked(slice) }; + // SAFETY: Lemma: In the following safety arguments, note that `slice` + // is derived from `self` in two steps: first, by casting `self: [T; N]` + // to `start: T`, then by constructing a pointer to a slice starting at + // `start` of length `N`. As a result, `slice` references exactly the + // same allocation as `self`, if any. + // + // 0. By the above lemma, if `slice`'s referent is not zero sized, then + // `slice` has the same referent as `self`. By invariant on `self`, + // this referent is entirely contained within some allocation, `A`. + // Because `slice` was constructed using provenance-preserving + // operations, it has provenance for its entire referent. + // 1. By the above lemma, if `slice`'s referent is not zero sized, then + // `A` is guaranteed to live for at least `'a`, because it is derived + // from the same allocation as `self`, which, by invariant on + // `PtrInner`, lives for at least `'a`. + unsafe { PtrInner::new(slice) } + } +} + +impl<'a> PtrInner<'a, [u8]> { + /// Attempts to cast `self` to a `U` using the given cast type. + /// + /// If `U` is a slice DST and pointer metadata (`meta`) is provided, then + /// the cast will only succeed if it would produce an object with the given + /// metadata. + /// + /// Returns `None` if the resulting `U` would be invalidly-aligned, if no + /// `U` can fit in `self`, or if the provided pointer metadata describes an + /// invalid instance of `U`. On success, returns a pointer to the + /// largest-possible `U` which fits in `self`. + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may rely + /// on that assumption for the soundness of their code. In particular, the + /// caller may assume that, if `try_cast_into` returns `Some((ptr, + /// remainder))`, then `ptr` and `remainder` refer to non-overlapping byte + /// ranges within `self`, and that `ptr` and `remainder` entirely cover + /// `self`. Finally: + /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a suffix cast, `remainder` has the same address as `self`. + #[inline] + pub fn try_cast_into( + self, + cast_type: CastType, + meta: Option, + ) -> Result<(PtrInner<'a, U>, PtrInner<'a, [u8]>), CastError> + where + U: 'a + ?Sized + KnownLayout, + { + // PANICS: By invariant, the byte range addressed by + // `self.as_non_null()` does not wrap around the address space. This + // implies that the sum of the address (represented as a `usize`) and + // length do not overflow `usize`, as required by + // `validate_cast_and_convert_metadata`. Thus, this call to + // `validate_cast_and_convert_metadata` will only panic if `U` is a DST + // whose trailing slice element is zero-sized. + let maybe_metadata = MetadataOf::::validate_cast_and_convert_metadata( + AsAddress::addr(self.as_ptr()), + self.meta(), + cast_type, + meta, + ); + + let (elems, split_at) = match maybe_metadata { + Ok((elems, split_at)) => (elems, split_at), + Err(MetadataCastError::Alignment) => { + // SAFETY: Since `validate_cast_and_convert_metadata` returned + // an alignment error, `U` must have an alignment requirement + // greater than one. + let err = unsafe { AlignmentError::<_, U>::new_unchecked(self) }; + return Err(CastError::Alignment(err)); + } + Err(MetadataCastError::Size) => return Err(CastError::Size(SizeError::new(self))), + }; + + // SAFETY: `validate_cast_and_convert_metadata` promises to return + // `split_at <= self.meta()`. + // + // Lemma 0: `l_slice` and `r_slice` are non-overlapping. Proof: By + // contract on `PtrInner::split_at_unchecked`, the produced `PtrInner`s + // are always non-overlapping if `self` is a `[T]`; here it is a `[u8]`. + let (l_slice, r_slice) = unsafe { self.split_at_unchecked(split_at) }; + + let (target, remainder) = match cast_type { + CastType::Prefix => (l_slice, r_slice), + CastType::Suffix => (r_slice, l_slice), + }; + + let base = target.as_non_null().cast::(); + + let ptr = U::raw_from_ptr_len(base, elems.get()); + + // SAFETY: + // 0. By invariant, if `target`'s referent is not zero sized, then + // `target` has provenance valid for some Rust allocation, `A`. + // Because `ptr` is derived from `target` via provenance-preserving + // operations, `ptr` will also have provenance valid for its entire + // referent. + // 1. `validate_cast_and_convert_metadata` promises that the object + // described by `elems` and `split_at` lives at a byte range which is + // a subset of the input byte range. Thus, by invariant, if + // `target`'s referent is not zero sized, then `target` refers to an + // allocation which is guaranteed to live for at least `'a`, and thus + // so does `ptr`. + Ok((unsafe { PtrInner::new(ptr) }, remainder)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::*; + + #[test] + fn test_meta() { + let arr = [1; 16]; + let dst = <[u8]>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + assert_eq!(ptr.meta().get(), 16); + + // SAFETY: 8 is less than 16 + let ptr = unsafe { ptr.with_meta(8) }; + + assert_eq!(ptr.meta().get(), 8); + } + + #[test] + fn test_split_at() { + fn test_split_at() { + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct SliceDst { + prefix: [u8; OFFSET], + trailing: [u8], + } + + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + for i in 0..=n { + assert_eq!(ptr.meta().get(), n); + // SAFETY: `i` is in bounds by construction. + let i = unsafe { MetadataOf::new_unchecked(i) }; + // SAFETY: `i` is in bounds by construction. + let (l, r) = unsafe { ptr.split_at_unchecked(i) }; + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let l_sum: usize = l + .trailing_slice() + .iter() + .map( + #[inline(always)] + |ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize, + ) + .sum(); + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let r_sum: usize = r + .iter() + .map( + #[inline(always)] + |ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize, + ) + .sum(); + assert_eq!(l_sum, i.get()); + assert_eq!(r_sum, n - i.get()); + assert_eq!(l_sum + r_sum, n); + } + } + + test_split_at::<0, 16>(); + test_split_at::<1, 17>(); + test_split_at::<2, 18>(); + } + + #[test] + fn test_trailing_slice() { + fn test_trailing_slice() { + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct SliceDst { + prefix: [u8; OFFSET], + trailing: [u8], + } + + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + + assert_eq!(ptr.meta().get(), n); + let trailing = ptr.trailing_slice(); + assert_eq!(trailing.meta().get(), n); + + assert_eq!( + // SAFETY: We assume this to be sound for the sake of this test, + // which will fail, here, in miri, if the safety precondition of + // `offset_of` is not satisfied. + unsafe { + #[allow(clippy::as_conversions)] + let offset = (trailing.as_ptr() as *mut u8).offset_from(ptr.as_ptr() as *mut _); + offset + }, + isize::try_from(OFFSET).unwrap(), + ); + + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let trailing: usize = trailing + .iter() + .map(|ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize) + .sum(); + + assert_eq!(trailing, n); + } + + test_trailing_slice::<0, 16>(); + test_trailing_slice::<1, 17>(); + test_trailing_slice::<2, 18>(); + } + #[test] + fn test_ptr_inner_clone() { + let mut x = 0u8; + let p = PtrInner::from_mut(&mut x); + #[allow(clippy::clone_on_copy)] + let p2 = p.clone(); + assert_eq!(p.as_non_null(), p2.as_non_null()); + } +} diff --git a/rust/zerocopy/src/pointer/invariant.rs b/rust/zerocopy/src/pointer/invariant.rs new file mode 100644 index 000000000000..2af211dd9fdd --- /dev/null +++ b/rust/zerocopy/src/pointer/invariant.rs @@ -0,0 +1,296 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_copy_implementations, missing_debug_implementations, missing_docs)] + +//! The parameterized invariants of a [`Ptr`][super::Ptr]. +//! +//! Invariants are encoded as ([`Aliasing`], [`Alignment`], [`Validity`]) +//! triples implementing the [`Invariants`] trait. + +/// The invariants of a [`Ptr`][super::Ptr]. +pub trait Invariants: Sealed { + type Aliasing: Aliasing; + type Alignment: Alignment; + type Validity: Validity; +} + +impl Invariants for (A, AA, V) { + type Aliasing = A; + type Alignment = AA; + type Validity = V; +} + +/// The aliasing invariant of a [`Ptr`][super::Ptr]. +/// +/// All aliasing invariants must permit reading from the bytes of a pointer's +/// referent which are not covered by [`UnsafeCell`]s. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +pub trait Aliasing: Sealed { + /// Is `Self` [`Exclusive`]? + #[doc(hidden)] + const IS_EXCLUSIVE: bool; +} + +/// The alignment invariant of a [`Ptr`][super::Ptr]. +pub trait Alignment: Sealed { + #[doc(hidden)] + #[must_use] + fn read(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read, + I: Invariants, + I::Aliasing: Reference; +} + +/// The validity invariant of a [`Ptr`][super::Ptr]. +/// +/// # Safety +/// +/// In this section, we will use `Ptr` as a shorthand for `Ptr>` for brevity. +/// +/// Each `V: Validity` defines a set of bit values which may appear in the +/// referent of a `Ptr`, denoted `S(T, V)`. Each `V: Validity`, in its +/// documentation, provides a definition of `S(T, V)` which must be valid for +/// all `T: ?Sized`. Any `V: Validity` must guarantee that this set is only a +/// function of the *bit validity* of the referent type, `T`, and not of any +/// other property of `T`. As a consequence, given `V: Validity`, `T`, and `U` +/// where `T` and `U` have the same bit validity, `S(V, T) = S(V, U)`. +/// +/// It is guaranteed that the referent of any `ptr: Ptr` is a member of +/// `S(T, V)`. Unsafe code must ensure that this guarantee will be upheld for +/// any existing `Ptr`s or any `Ptr`s that that code creates. +/// +/// An important implication of this guarantee is that it restricts what +/// transmutes are sound, where "transmute" is used in this context to refer to +/// changing the referent type or validity invariant of a `Ptr`, as either +/// change may change the set of bit values permitted to appear in the referent. +/// In particular, the following are necessary (but not sufficient) conditions +/// in order for a transmute from `src: Ptr` to `dst: Ptr` to be +/// sound: +/// - If `S(T, V) = S(U, W)`, then no restrictions apply; otherwise, +/// - If `dst` permits mutation of its referent (e.g. via `Exclusive` aliasing +/// or interior mutation under `Shared` aliasing), then it must hold that +/// `S(T, V) ⊇ S(U, W)` - in other words, the transmute must not expand the +/// set of allowed referent bit patterns. A violation of this requirement +/// would permit using `dst` to write `x` where `x ∈ S(U, W)` but `x ∉ S(T, +/// V)`, which would violate the guarantee that `src`'s referent may only +/// contain values in `S(T, V)`. +/// - If the referent may be mutated without going through `dst` while `dst` is +/// live (e.g. via interior mutation on a `Shared`-aliased `Ptr` or `&` +/// reference), then it must hold that `S(T, V) ⊆ S(U, W)` - in other words, +/// the transmute must not shrink the set of allowed referent bit patterns. A +/// violation of this requirement would permit using `src` or another +/// mechanism (e.g. a `&` reference used to derive `src`) to write `x` where +/// `x ∈ S(T, V)` but `x ∉ S(U, W)`, which would violate the guarantee that +/// `dst`'s referent may only contain values in `S(U, W)`. +pub unsafe trait Validity: Sealed { + const KIND: ValidityKind; +} + +pub enum ValidityKind { + Uninit, + AsInitialized, + Initialized, + Valid, +} + +/// An [`Aliasing`] invariant which is either [`Shared`] or [`Exclusive`]. +/// +/// # Safety +/// +/// Given `A: Reference`, callers may assume that either `A = Shared` or `A = +/// Exclusive`. +pub trait Reference: Aliasing + Sealed {} + +/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a T`. +/// +/// The referent of a shared-aliased `Ptr` may be concurrently referenced by any +/// number of shared-aliased `Ptr` or `&T` references, or by any number of +/// `Ptr` or `&U` references as permitted by `T`'s library safety invariants, +/// and may not be concurrently referenced by any exclusively-aliased `Ptr`s or +/// `&mut` references. The referent must not be mutated, except via +/// [`UnsafeCell`]s, and only when permitted by `T`'s library safety invariants. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +pub enum Shared {} +impl Aliasing for Shared { + const IS_EXCLUSIVE: bool = false; +} +impl Reference for Shared {} + +/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a mut T`. +/// +/// The referent of an exclusively-aliased `Ptr` may not be concurrently +/// referenced by any other `Ptr`s or references, and may not be accessed (read +/// or written) other than via this `Ptr`. +pub enum Exclusive {} +impl Aliasing for Exclusive { + const IS_EXCLUSIVE: bool = true; +} +impl Reference for Exclusive {} + +/// It is unknown whether the pointer is aligned. +pub enum Unaligned {} + +impl Alignment for Unaligned { + #[inline(always)] + fn read(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read, + I: Invariants, + I::Aliasing: Reference, + { + (*ptr.into_unalign().as_ref()).into_inner() + } +} + +/// The referent is aligned: for `Ptr`, the referent's address is a multiple +/// of the `T`'s alignment. +pub enum Aligned {} +impl Alignment for Aligned { + #[inline(always)] + fn read(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read, + I: Invariants, + I::Aliasing: Reference, + { + *ptr.as_ref() + } +} + +/// Any bit pattern is allowed in the `Ptr`'s referent, including uninitialized +/// bytes. +pub enum Uninit {} +// SAFETY: `Uninit`'s validity is well-defined for all `T: ?Sized`, and is not a +// function of any property of `T` other than its bit validity (in fact, it's +// not even a property of `T`'s bit validity, but this is more than we are +// required to uphold). +unsafe impl Validity for Uninit { + const KIND: ValidityKind = ValidityKind::Uninit; +} + +/// The byte ranges initialized in `T` are also initialized in the referent of a +/// `Ptr`. +/// +/// Formally: uninitialized bytes may only be present in `Ptr`'s referent +/// where they are guaranteed to be present in `T`. This is a dynamic property: +/// if, at a particular byte offset, a valid enum discriminant is set, the +/// subsequent bytes may only have uninitialized bytes as specified by the +/// corresponding enum. +/// +/// Formally, given `len = size_of_val_raw(ptr)`, at every byte offset, `b`, in +/// the range `[0, len)`: +/// - If, in any instance `t: T` of length `len`, the byte at offset `b` in `t` +/// is initialized, then the byte at offset `b` within `*ptr` must be +/// initialized. +/// - Let `c` be the contents of the byte range `[0, b)` in `*ptr`. Let `S` be +/// the subset of valid instances of `T` of length `len` which contain `c` in +/// the offset range `[0, b)`. If, in any instance of `t: T` in `S`, the byte +/// at offset `b` in `t` is initialized, then the byte at offset `b` in `*ptr` +/// must be initialized. +/// +/// Pragmatically, this means that if `*ptr` is guaranteed to contain an enum +/// type at a particular offset, and the enum discriminant stored in `*ptr` +/// corresponds to a valid variant of that enum type, then it is guaranteed +/// that the appropriate bytes of `*ptr` are initialized as defined by that +/// variant's bit validity (although note that the variant may contain another +/// enum type, in which case the same rules apply depending on the state of +/// its discriminant, and so on recursively). +pub enum AsInitialized {} +// SAFETY: `AsInitialized`'s validity is well-defined for all `T: ?Sized`, and +// is not a function of any property of `T` other than its bit validity. +unsafe impl Validity for AsInitialized { + const KIND: ValidityKind = ValidityKind::AsInitialized; +} + +/// The byte ranges in the referent are fully initialized. In other words, if +/// the referent is `N` bytes long, then it contains a bit-valid `[u8; N]`. +pub enum Initialized {} +// SAFETY: `Initialized`'s validity is well-defined for all `T: ?Sized`, and is +// not a function of any property of `T` other than its bit validity (in fact, +// it's not even a property of `T`'s bit validity, but this is more than we are +// required to uphold). +unsafe impl Validity for Initialized { + const KIND: ValidityKind = ValidityKind::Initialized; +} + +/// The referent of a `Ptr` is valid for `T`, upholding bit validity and any +/// library safety invariants. +pub enum Valid {} +// SAFETY: `Valid`'s validity is well-defined for all `T: ?Sized`, and is not a +// function of any property of `T` other than its bit validity. +unsafe impl Validity for Valid { + const KIND: ValidityKind = ValidityKind::Valid; +} + +/// # Safety +/// +/// `DT: CastableFrom` is sound if `SV = DV = Uninit` or `SV = DV = +/// Initialized`. +pub unsafe trait CastableFrom {} + +// SAFETY: `SV = DV = Uninit`. +unsafe impl CastableFrom for DT {} +// SAFETY: `SV = DV = Initialized`. +unsafe impl CastableFrom for DT {} + +/// [`Ptr`](crate::Ptr) referents that permit unsynchronized read operations. +/// +/// `T: Read` implies that a pointer to `T` with aliasing `A` permits +/// unsynchronized read operations. This can be because `A` is [`Exclusive`] or +/// because `T` does not permit interior mutation. +/// +/// # Safety +/// +/// `T: Read` if either of the following conditions holds: +/// - `A` is [`Exclusive`] +/// - `T` implements [`Immutable`](crate::Immutable) +/// +/// As a consequence, if `T: Read`, then any `Ptr` is +/// permitted to perform unsynchronized reads from its referent. +pub trait Read {} + +impl Read for T {} +impl Read for T {} + +/// Unsynchronized reads are permitted because only one live [`Ptr`](crate::Ptr) +/// or reference may exist to the referent bytes at a time. +#[derive(Copy, Clone, Debug)] +pub enum BecauseExclusive {} + +/// Unsynchronized reads are permitted because no live [`Ptr`](crate::Ptr)s or +/// references permit interior mutation. +#[derive(Copy, Clone, Debug)] +pub enum BecauseImmutable {} + +use sealed::Sealed; +mod sealed { + use super::*; + + pub trait Sealed {} + + impl Sealed for Shared {} + impl Sealed for Exclusive {} + + impl Sealed for Unaligned {} + impl Sealed for Aligned {} + + impl Sealed for Uninit {} + impl Sealed for AsInitialized {} + impl Sealed for Initialized {} + impl Sealed for Valid {} + + impl Sealed for (A, AA, V) {} + + impl Sealed for BecauseImmutable {} + impl Sealed for BecauseExclusive {} +} diff --git a/rust/zerocopy/src/pointer/mod.rs b/rust/zerocopy/src/pointer/mod.rs new file mode 100644 index 000000000000..4c3658679d46 --- /dev/null +++ b/rust/zerocopy/src/pointer/mod.rs @@ -0,0 +1,408 @@ +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Abstractions over raw pointers. + +#![allow(missing_docs)] + +mod inner; +pub mod invariant; +mod ptr; +pub mod transmute; + +pub use inner::PtrInner; +pub use invariant::{BecauseExclusive, BecauseImmutable, Read}; +pub use ptr::{Ptr, TryWithError}; +pub use transmute::*; + +use crate::wrappers::ReadOnly; + +/// A shorthand for a maybe-valid, maybe-aligned reference. Used as the argument +/// to [`TryFromBytes::is_bit_valid`]. +/// +/// [`TryFromBytes::is_bit_valid`]: crate::TryFromBytes::is_bit_valid +pub type Maybe<'a, T, Alignment = invariant::Unaligned> = + Ptr<'a, ReadOnly, (invariant::Shared, Alignment, invariant::Initialized)>; + +/// Checks if the referent is zeroed. +pub(crate) fn is_zeroed(ptr: Ptr<'_, T, I>) -> bool +where + T: crate::Immutable + crate::KnownLayout, + I: invariant::Invariants, + I::Aliasing: invariant::Reference, +{ + ptr.as_bytes().as_ref().iter().all( + #[inline(always)] + |&byte| byte == 0, + ) +} + +pub mod cast { + use core::{marker::PhantomData, mem}; + + use crate::{ + layout::{SizeInfo, TrailingSliceLayout}, + HasField, KnownLayout, PtrInner, + }; + + /// A pointer cast or projection. + /// + /// # Safety + /// + /// The implementation of `project` must satisfy its safety post-condition. + pub unsafe trait Project { + /// Projects a pointer from `Src` to `Dst`. + /// + /// Users should generally not call `project` directly, and instead + /// should use high-level APIs like [`PtrInner::project`] or + /// [`Ptr::project`]. + /// + /// [`Ptr::project`]: crate::pointer::Ptr::project + /// + /// # Safety + /// + /// The returned pointer refers to a non-strict subset of the bytes of + /// `src`'s referent, and has the same provenance as `src`. + fn project(src: PtrInner<'_, Src>) -> *mut Dst; + } + + /// A [`Project`] which preserves the address of the referent – a pointer + /// cast. + /// + /// # Safety + /// + /// A `Cast` projection must preserve the address of the referent. It may + /// shrink the set of referent bytes, and it may change the referent's type. + pub unsafe trait Cast: Project {} + + /// A [`Cast`] which does not shrink the set of referent bytes. + /// + /// # Safety + /// + /// A `CastExact` projection must preserve the set of referent bytes. + pub unsafe trait CastExact: Cast {} + + /// A no-op pointer cast. + #[derive(Default, Copy, Clone)] + #[allow(missing_debug_implementations)] + pub struct IdCast; + + // SAFETY: `project` returns its argument unchanged, and so it is a + // provenance-preserving projection which preserves the set of referent + // bytes. + unsafe impl Project for IdCast { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut T { + src.as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl Cast for IdCast {} + + // SAFETY: The `Project::project` impl preserves referent size. + unsafe impl CastExact for IdCast {} + + /// A pointer cast which preserves or shrinks the set of referent bytes of + /// a statically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that `Dst` is no larger than `Src`. Thus, `CastSized` has a + /// sound implementation of [`Project`] for all `Src` and `Dst` – the caller + /// may pass any `Src` and `Dst` without being responsible for soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastSized {} + + // SAFETY: By the `static_assert!`, `Dst` is no larger than `Src`, + // and so all casts preserve or shrink the set of referent bytes. All + // operations preserve provenance. + unsafe impl Project for CastSized { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + static_assert!(Src, Dst => mem::size_of::() >= mem::size_of::()); + src.as_ptr().cast::() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl Cast for CastSized {} + + /// A pointer cast which preserves the set of referent bytes of a + /// statically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that `Dst` has the same size as `Src`. Thus, `CastSizedExact` + /// has a sound implementation of [`Project`] for all `Src` and `Dst` – the + /// caller may pass any `Src` and `Dst` without being responsible for + /// soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastSizedExact {} + + // SAFETY: By the `static_assert!`, `Dst` has the same size as `Src`, + // and so all casts preserve the set of referent bytes. All operations + // preserve provenance. + unsafe impl Project for CastSizedExact { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + static_assert!(Src, Dst => mem::size_of::() == mem::size_of::()); + src.as_ptr().cast::() + } + } + + // SAFETY: The `Project::project_raw` impl preserves referent address. + unsafe impl Cast for CastSizedExact {} + + // SAFETY: By the `static_assert!`, `Project::project_raw` impl preserves + // referent size. + unsafe impl CastExact for CastSizedExact {} + + /// A pointer cast which preserves or shrinks the set of referent bytes of + /// a dynamically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that the cast preserves the set of referent bytes. Thus, + /// `CastUnsized` has a sound implementation of [`Project`] for all `Src` + /// and `Dst` – the caller may pass any `Src` and `Dst` without being + /// responsible for soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastUnsized {} + + // SAFETY: By the `static_assert!`, `Src` and `Dst` are either: + // - Both sized and equal in size + // - Both slice DSTs with the same trailing slice offset and element size + // and with align_of::() == align_of::(). These ensure that any + // given pointer metadata encodes the same size for both `Src` and `Dst` + // (note that the alignment is required as it affects the amount of + // trailing padding). Thus, `project` preserves the set of referent bytes. + unsafe impl Project for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout, + { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + // FIXME: Do we want this to support shrinking casts as well? If so, + // we'll need to remove the `CastExact` impl. + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + let src = ::LAYOUT; + let dst = ::LAYOUT; + match (src.size_info, dst.size_info) { + (SizeInfo::Sized { size: src_size }, SizeInfo::Sized { size: dst_size }) => src_size == dst_size, + ( + SizeInfo::SliceDst(TrailingSliceLayout { offset: src_offset, elem_size: src_elem_size }), + SizeInfo::SliceDst(TrailingSliceLayout { offset: dst_offset, elem_size: dst_elem_size }) + ) => src.align.get() == dst.align.get() && src_offset == dst_offset && src_elem_size == dst_elem_size, + _ => false, + } + }); + + let metadata = Src::pointer_to_metadata(src.as_ptr()); + Dst::raw_from_ptr_len(src.as_non_null().cast::(), metadata).as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl Cast for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout, + { + } + + // SAFETY: By the `static_assert!` in `Project::project`, `Src` and `Dst` + // are either: + // - Both sized and equal in size + // - Both slice DSTs with the same alignment, trailing slice offset, and + // element size. These ensure that any given pointer metadata encodes the + // same size for both `Src` and `Dst` (note that the alignment is required + // as it affects the amount of trailing padding). + unsafe impl CastExact for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout, + { + } + + /// A field projection + /// + /// A `Projection` is a [`Project`] which implements projection by + /// delegating to an implementation of [`HasField::project`]. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub struct Projection { + _never: core::convert::Infallible, + _phantom: PhantomData, + } + + // SAFETY: `HasField::project` has the same safety post-conditions as + // `Project::project`. + unsafe impl Project + for Projection + where + T: HasField, + { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut T::Type { + T::project(src) + } + } + + // SAFETY: All `repr(C)` union fields exist at offset 0 within the union [1], + // and so any union projection is actually a cast (ie, preserves address). + // + // [1] Per + // https://doc.rust-lang.org/1.92.0/reference/type-layout.html#reprc-unions, + // it's not *technically* guaranteed that non-maximally-sized fields + // are at offset 0, but it's clear that this is the intention of `repr(C)` + // unions. It says: + // + // > A union declared with `#[repr(C)]` will have the same size and + // > alignment as an equivalent C union declaration in the C language for + // > the target platform. + // + // Note that this only mentions size and alignment, not layout. However, + // C unions *do* guarantee that all fields start at offset 0. [2] + // + // This is also reinforced by + // https://doc.rust-lang.org/1.92.0/reference/items/unions.html#r-items.union.fields.offset: + // + // > Fields might have a non-zero offset (except when the C + // > representation is used); in that case the bits starting at the + // > offset of the fields are read + // + // [2] Per https://port70.net/~nsz/c/c11/n1570.html#6.7.2.1p16: + // + // > The size of a union is sufficient to contain the largest of its + // > members. The value of at most one of the members can be stored in a + // > union object at any time. A pointer to a union object, suitably + // > converted, points to each of its members (or if a member is a + // > bit-field, then to the unit in which it resides), and vice versa. + // + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/595): + // Cite the documentation once it's updated. + unsafe impl Cast + for Projection + where + T: HasField, + { + } + + /// A transitive sequence of projections. + /// + /// Given `TU: Project` and `UV: Project`, `TransitiveProject<_, TU, UV>` is + /// a [`Project`] which projects by applying `TU` followed by `UV`. + /// + /// If `TU: Cast` and `UV: Cast`, then `TransitiveProject<_, TU, UV>: Cast`. + #[allow(missing_debug_implementations)] + pub struct TransitiveProject { + _never: core::convert::Infallible, + _projections: PhantomData<(TU, UV)>, + // On our MSRV (1.56), the debuginfo for a tuple containing both an + // uninhabited type and a DST causes an ICE. We split `U` from `TU` and + // `UV` to avoid this situation. + _u: PhantomData, + } + + // SAFETY: Since `TU::project` and `UV::project` are each + // provenance-preserving operations which preserve or shrink the set of + // referent bytes, so is their composition. + unsafe impl Project for TransitiveProject + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: Project, + UV: Project, + { + #[inline(always)] + fn project(t: PtrInner<'_, T>) -> *mut V { + t.project::<_, TU>().project::<_, UV>().as_ptr() + } + } + + // SAFETY: Since the `Project::project` impl delegates to `TU::project` and + // `UV::project`, and since `TU` and `UV` are `Cast`, the `Project::project` + // impl preserves the address of the referent. + unsafe impl Cast for TransitiveProject + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: Cast, + UV: Cast, + { + } + + // SAFETY: Since the `Project::project` impl delegates to `TU::project` and + // `UV::project`, and since `TU` and `UV` are `CastExact`, the `Project::project` + // impl preserves the set of referent bytes. + unsafe impl CastExact for TransitiveProject + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: CastExact, + UV: CastExact, + { + } + + /// A cast from `T` to `[u8]`. + #[allow(missing_copy_implementations, missing_debug_implementations)] + pub struct AsBytesCast; + + // SAFETY: `project` constructs a pointer with the same address as `src` + // and with a referent of the same size as `*src`. It does this using + // provenance-preserving operations. + // + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/594): + // Technically, this proof assumes that `*src` is contiguous (the same is + // true of other proofs in this codebase). Is this guaranteed anywhere? + unsafe impl Project for AsBytesCast { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut [u8] { + let bytes = match T::size_of_val_raw(src.as_non_null()) { + Some(bytes) => bytes, + // SAFETY: `KnownLayout::size_of_val_raw` promises to always + // return `Some` so long as the resulting size fits in a + // `usize`. By invariant on `PtrInner`, `src` refers to a range + // of bytes whose size fits in an `isize`, which implies that it + // also fits in a `usize`. + None => unsafe { core::hint::unreachable_unchecked() }, + }; + + core::ptr::slice_from_raw_parts_mut(src.as_ptr().cast::(), bytes) + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl Cast for AsBytesCast {} + + // SAFETY: The `Project::project` impl preserves the set of referent bytes. + unsafe impl CastExact for AsBytesCast {} + + /// A cast from any type to `()`. + #[allow(missing_copy_implementations, missing_debug_implementations)] + pub struct CastToUnit; + + // SAFETY: The `project` implementation projects to a subset of its + // argument's referent using provenance-preserving operations. + unsafe impl Project for CastToUnit { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut () { + src.as_ptr().cast::<()>() + } + } + + // SAFETY: The `project` implementation preserves referent address. + unsafe impl Cast for CastToUnit {} +} diff --git a/rust/zerocopy/src/pointer/ptr.rs b/rust/zerocopy/src/pointer/ptr.rs new file mode 100644 index 000000000000..307b2aee63ca --- /dev/null +++ b/rust/zerocopy/src/pointer/ptr.rs @@ -0,0 +1,1584 @@ +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_docs)] + +use core::{ + fmt::{Debug, Formatter}, + marker::PhantomData, +}; + +use crate::{ + pointer::{ + inner::PtrInner, + invariant::*, + transmute::{MutationCompatible, SizeEq, TransmuteFromPtr}, + }, + AlignmentError, CastError, CastType, KnownLayout, SizeError, TryFromBytes, ValidityError, +}; + +/// Module used to gate access to [`Ptr`]'s fields. +mod def { + #[cfg(doc)] + use super::super::invariant; + use super::*; + + /// A raw pointer with more restrictions. + /// + /// `Ptr` is similar to [`NonNull`], but it is more restrictive in the + /// following ways (note that these requirements only hold of non-zero-sized + /// referents): + /// - It must derive from a valid allocation. + /// - It must reference a byte range which is contained inside the + /// allocation from which it derives. + /// - As a consequence, the byte range it references must have a size + /// which does not overflow `isize`. + /// + /// Depending on how `Ptr` is parameterized, it may have additional + /// invariants: + /// - `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// - `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// - `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + /// + /// `Ptr<'a, T>` is [covariant] in `'a` and invariant in `T`. + /// + /// [`NonNull`]: core::ptr::NonNull + /// [covariant]: https://doc.rust-lang.org/reference/subtyping.html + pub struct Ptr<'a, T, I> + where + T: ?Sized, + I: Invariants, + { + /// # Invariants + /// + /// 0. `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// 1. `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// 2. `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + // SAFETY: `PtrInner<'a, T>` is covariant in `'a` and invariant in `T`. + ptr: PtrInner<'a, T>, + _invariants: PhantomData, + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Constructs a new `Ptr` from a [`PtrInner`]. + /// + /// # Safety + /// + /// The caller promises that: + /// + /// 0. `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// 1. `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// 2. `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + pub(crate) unsafe fn from_inner(ptr: PtrInner<'a, T>) -> Ptr<'a, T, I> { + // SAFETY: The caller has promised to satisfy all safety invariants + // of `Ptr`. + Self { ptr, _invariants: PhantomData } + } + + /// Converts this `Ptr` to a [`PtrInner`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned value in a way + /// that violates the safety invariants of `self`. + #[inline] + #[must_use] + pub fn as_inner(&self) -> PtrInner<'a, T> { + self.ptr + } + } +} + +#[allow(unreachable_pub)] // This is a false positive on our MSRV toolchain. +pub use def::Ptr; + +/// External trait implementations on [`Ptr`]. +mod _external { + use super::*; + + /// SAFETY: Shared pointers are safely `Copy`. `Ptr`'s other invariants + /// (besides aliasing) are unaffected by the number of references that exist + /// to `Ptr`'s referent. The notable cases are: + /// - Alignment is a property of the referent type (`T`) and the address, + /// both of which are unchanged + /// - Let `S(T, V)` be the set of bit values permitted to appear in the + /// referent of a `Ptr>`. Since this copy + /// does not change `I::Validity` or `T`, `S(T, I::Validity)` is also + /// unchanged. + /// + /// We are required to guarantee that the referents of the original `Ptr` + /// and of the copy (which, of course, are actually the same since they + /// live in the same byte address range) both remain in the set `S(T, + /// I::Validity)`. Since this invariant holds on the original `Ptr`, it + /// cannot be violated by the original `Ptr`, and thus the original `Ptr` + /// cannot be used to violate this invariant on the copy. The inverse + /// holds as well. + impl<'a, T, I> Copy for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + } + + /// SAFETY: See the safety comment on `Copy`. + impl<'a, T, I> Clone for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + #[inline] + fn clone(&self) -> Self { + *self + } + } + + impl<'a, T, I> Debug for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + self.as_inner().as_non_null().fmt(f) + } + } +} + +/// Methods for converting to and from `Ptr` and Rust's safe reference types. +mod _conversions { + use super::*; + use crate::pointer::cast::{CastExact, CastSized, IdCast}; + + /// `&'a T` → `Ptr<'a, T>` + impl<'a, T> Ptr<'a, T, (Shared, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Constructs a `Ptr` from a shared reference. + #[inline(always)] + pub fn from_ref(ptr: &'a T) -> Self { + let inner = PtrInner::from_ref(ptr); + // SAFETY: + // 0. `ptr`, by invariant on `&'a T`, conforms to the aliasing + // invariant of `Shared`. + // 1. `ptr`, by invariant on `&'a T`, conforms to the alignment + // invariant of `Aligned`. + // 2. `ptr`'s referent, by invariant on `&'a T`, is a bit-valid `T`. + // This satisfies the requirement that a `Ptr` + // point to a bit-valid `T`. Even if `T` permits interior + // mutation, this invariant guarantees that the returned `Ptr` + // can only ever be used to modify the referent to store + // bit-valid `T`s, which ensures that the returned `Ptr` cannot + // be used to violate the soundness of the original `ptr: &'a T` + // or of any other references that may exist to the same + // referent. + unsafe { Self::from_inner(inner) } + } + } + + /// `&'a mut T` → `Ptr<'a, T>` + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Constructs a `Ptr` from an exclusive reference. + #[inline(always)] + pub fn from_mut(ptr: &'a mut T) -> Self { + let inner = PtrInner::from_mut(ptr); + // SAFETY: + // 0. `ptr`, by invariant on `&'a mut T`, conforms to the aliasing + // invariant of `Exclusive`. + // 1. `ptr`, by invariant on `&'a mut T`, conforms to the alignment + // invariant of `Aligned`. + // 2. `ptr`'s referent, by invariant on `&'a mut T`, is a bit-valid + // `T`. This satisfies the requirement that a `Ptr` point to a bit-valid `T`. This invariant guarantees + // that the returned `Ptr` can only ever be used to modify the + // referent to store bit-valid `T`s, which ensures that the + // returned `Ptr` cannot be used to violate the soundness of the + // original `ptr: &'a mut T`. + unsafe { Self::from_inner(inner) } + } + } + + /// `Ptr<'a, T>` → `&'a T` + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + I::Aliasing: Reference, + { + /// Converts `self` to a shared reference. + // This consumes `self`, not `&self`, because `self` is, logically, a + // pointer. For `I::Aliasing = invariant::Shared`, `Self: Copy`, and so + // this doesn't prevent the caller from still using the pointer after + // calling `as_ref`. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_ref(self) -> &'a T { + let raw = self.as_inner().as_non_null(); + // SAFETY: `self` satisfies the `Aligned` invariant, so we know that + // `raw` is validly-aligned for `T`. + #[cfg(miri)] + unsafe { + crate::util::miri_promise_symbolic_alignment( + raw.as_ptr().cast(), + core::mem::align_of_val_raw(raw.as_ptr()), + ); + } + // SAFETY: This invocation of `NonNull::as_ref` satisfies its + // documented safety preconditions: + // + // 1. The pointer is properly aligned. This is ensured by-contract + // on `Ptr`, because the `I::Alignment` is `Aligned`. + // + // 2. If the pointer's referent is not zero-sized, then the pointer + // must be “dereferenceable” in the sense defined in the module + // documentation; i.e.: + // + // > The memory range of the given size starting at the pointer + // > must all be within the bounds of a single allocated object. + // > [2] + // + // This is ensured by contract on all `PtrInner`s. + // + // 3. The pointer must point to a validly-initialized instance of + // `T`. This is ensured by-contract on `Ptr`, because the + // `I::Validity` is `Valid`. + // + // 4. You must enforce Rust’s aliasing rules. This is ensured by + // contract on `Ptr`, because `I::Aliasing: Reference`. Either it + // is `Shared` or `Exclusive`. If it is `Shared`, other + // references may not mutate the referent outside of + // `UnsafeCell`s. + // + // [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_ref + // [2]: https://doc.rust-lang.org/std/ptr/index.html#safety + unsafe { raw.as_ref() } + } + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + I::Aliasing: Reference, + { + /// Reborrows `self`, producing another `Ptr`. + /// + /// Since `self` is borrowed mutably, this prevents any methods from + /// being called on `self` as long as the returned `Ptr` exists. + #[inline] + #[must_use] + #[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below. + pub fn reborrow<'b>(&'b mut self) -> Ptr<'b, T, I> + where + 'a: 'b, + { + // SAFETY: The following all hold by invariant on `self`, and thus + // hold of `ptr = self.as_inner()`: + // 0. SEE BELOW. + // 1. `ptr` conforms to the alignment invariant of + // [`I::Alignment`](invariant::Alignment). + // 2. `ptr` conforms to the validity invariant of + // [`I::Validity`](invariant::Validity). `self` and the returned + // `Ptr` permit the same bit values in their referents since they + // have the same referent type (`T`) and the same validity + // (`I::Validity`). Thus, regardless of what mutation is + // permitted (`Exclusive` aliasing or `Shared`-aliased interior + // mutation), neither can be used to write a value to the + // referent which violates the other's validity invariant. + // + // For aliasing (0 above), since `I::Aliasing: Reference`, + // there are two cases for `I::Aliasing`: + // - For `invariant::Shared`: `'a` outlives `'b`, and so the + // returned `Ptr` does not permit accessing the referent any + // longer than is possible via `self`. For shared aliasing, it is + // sound for multiple `Ptr`s to exist simultaneously which + // reference the same memory, so creating a new one is not + // problematic. + // - For `invariant::Exclusive`: Since `self` is `&'b mut` and we + // return a `Ptr` with lifetime `'b`, `self` is inaccessible to + // the caller for the lifetime `'b` - in other words, `self` is + // inaccessible to the caller as long as the returned `Ptr` + // exists. Since `self` is an exclusive `Ptr`, no other live + // references or `Ptr`s may exist which refer to the same memory + // while `self` is live. Thus, as long as the returned `Ptr` + // exists, no other references or `Ptr`s which refer to the same + // memory may be live. + unsafe { Ptr::from_inner(self.as_inner()) } + } + + /// Reborrows `self` as shared, producing another `Ptr` with `Shared` + /// aliasing. + /// + /// Since `self` is borrowed mutably, this prevents any methods from + /// being called on `self` as long as the returned `Ptr` exists. + #[inline] + #[must_use] + #[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below. + pub fn reborrow_shared<'b>(&'b mut self) -> Ptr<'b, T, (Shared, I::Alignment, I::Validity)> + where + 'a: 'b, + { + // SAFETY: The following all hold by invariant on `self`, and thus + // hold of `ptr = self.as_inner()`: + // 0. SEE BELOW. + // 1. `ptr` conforms to the alignment invariant of + // [`I::Alignment`](invariant::Alignment). + // 2. `ptr` conforms to the validity invariant of + // [`I::Validity`](invariant::Validity). `self` and the returned + // `Ptr` permit the same bit values in their referents since they + // have the same referent type (`T`) and the same validity + // (`I::Validity`). Thus, regardless of what mutation is + // permitted (`Exclusive` aliasing or `Shared`-aliased interior + // mutation), neither can be used to write a value to the + // referent which violates the other's validity invariant. + // + // For aliasing (0 above), since `I::Aliasing: Reference`, + // there are two cases for `I::Aliasing`: + // - For `invariant::Shared`: `'a` outlives `'b`, and so the + // returned `Ptr` does not permit accessing the referent any + // longer than is possible via `self`. For shared aliasing, it is + // sound for multiple `Ptr`s to exist simultaneously which + // reference the same memory, so creating a new one is not + // problematic. + // - For `invariant::Exclusive`: Since `self` is `&'b mut` and we + // return a `Ptr` with lifetime `'b`, `self` is inaccessible to + // the caller for the lifetime `'b` - in other words, `self` is + // inaccessible to the caller as long as the returned `Ptr` + // exists. Since `self` is an exclusive `Ptr`, no other live + // references or `Ptr`s may exist which refer to the same memory + // while `self` is live. Thus, as long as the returned `Ptr` + // exists, no other references or `Ptr`s which refer to the same + // memory may be live. + unsafe { Ptr::from_inner(self.as_inner()) } + } + } + + /// `Ptr<'a, T>` → `&'a mut T` + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Converts `self` to a mutable reference. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_mut(self) -> &'a mut T { + let mut raw = self.as_inner().as_non_null(); + // SAFETY: `self` satisfies the `Aligned` invariant, so we know that + // `raw` is validly-aligned for `T`. + #[cfg(miri)] + unsafe { + crate::util::miri_promise_symbolic_alignment( + raw.as_ptr().cast(), + core::mem::align_of_val_raw(raw.as_ptr()), + ); + } + // SAFETY: This invocation of `NonNull::as_mut` satisfies its + // documented safety preconditions: + // + // 1. The pointer is properly aligned. This is ensured by-contract + // on `Ptr`, because the `ALIGNMENT_INVARIANT` is `Aligned`. + // + // 2. If the pointer's referent is not zero-sized, then the pointer + // must be “dereferenceable” in the sense defined in the module + // documentation; i.e.: + // + // > The memory range of the given size starting at the pointer + // > must all be within the bounds of a single allocated object. + // > [2] + // + // This is ensured by contract on all `PtrInner`s. + // + // 3. The pointer must point to a validly-initialized instance of + // `T`. This is ensured by-contract on `Ptr`, because the + // validity invariant is `Valid`. + // + // 4. You must enforce Rust’s aliasing rules. This is ensured by + // contract on `Ptr`, because the `ALIASING_INVARIANT` is + // `Exclusive`. + // + // [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_mut + // [2]: https://doc.rust-lang.org/std/ptr/index.html#safety + unsafe { raw.as_mut() } + } + } + + /// `Ptr<'a, T>` → `Ptr<'a, U>` + impl<'a, T: ?Sized, I> Ptr<'a, T, I> + where + I: Invariants, + { + #[must_use] + #[inline(always)] + pub fn transmute(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + U: TransmuteFromPtr>::CastFrom, R> + + SizeEq + + ?Sized, + { + self.transmute_with::>::CastFrom, R>() + } + + #[inline] + #[must_use] + pub fn transmute_with(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + U: TransmuteFromPtr + ?Sized, + C: CastExact, + { + // SAFETY: + // - By `C: CastExact`, `C` preserves referent address, and so we + // don't need to consider projections in the following safety + // arguments. + // - If aliasing is `Shared`, then by `U: TransmuteFromPtr`, at + // least one of the following holds: + // - `T: Immutable` and `U: Immutable`, in which case it is + // trivially sound for shared code to operate on a `&T` and `&U` + // at the same time, as neither can perform interior mutation + // - It is directly guaranteed that it is sound for shared code to + // operate on these references simultaneously + // - By `U: TransmuteFromPtr`, it + // is sound to perform this transmute using `C`. + unsafe { self.project_transmute_unchecked::<_, _, C>() } + } + + #[inline] + #[must_use] + pub fn recall_validity(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> + where + V: Validity, + T: TransmuteFromPtr, + { + let ptr = self.transmute_with::(); + // SAFETY: `self` and `ptr` have the same address and referent type. + // Therefore, if `self` satisfies `I::Alignment`, then so does + // `ptr`. + unsafe { ptr.assume_alignment::() } + } + + /// Projects and/or transmutes to a different (unsized) referent type + /// without checking interior mutability. + /// + /// Callers should prefer [`cast`] or [`project`] where possible. + /// + /// [`cast`]: Ptr::cast + /// [`project`]: Ptr::project + /// + /// # Safety + /// + /// The caller promises that: + /// - If `I::Aliasing` is [`Shared`], it must not be possible for safe + /// code, operating on a `&T` and `&U`, with the referents of `self` + /// and `self.project_transmute_unchecked()`, respectively, to cause + /// undefined behavior. + /// - It is sound to project and/or transmute a pointer of type `T` with + /// aliasing `I::Aliasing` and validity `I::Validity` to a pointer of + /// type `U` with aliasing `I::Aliasing` and validity `V`. This is a + /// subtle soundness requirement that is a function of `T`, `U`, + /// `I::Aliasing`, `I::Validity`, and `V`, and may depend upon the + /// presence, absence, or specific location of `UnsafeCell`s in `T` + /// and/or `U`, and on whether interior mutation is ever permitted via + /// those `UnsafeCell`s. See [`Validity`] for more details. + #[inline] + #[must_use] + pub unsafe fn project_transmute_unchecked( + self, + ) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + P: crate::pointer::cast::Project, + { + let ptr = self.as_inner().project::<_, P>(); + + // SAFETY: + // + // The following safety arguments rely on the fact that `P: Project` + // guarantees that `P` is a referent-preserving or -shrinking + // projection. Thus, `ptr` addresses a subset of the bytes of + // `*self`, and so certain properties that hold of `*self` also hold + // of `*ptr`. + // + // 0. `ptr` conforms to the aliasing invariant of `I::Aliasing`: + // - `Exclusive`: `self` is the only `Ptr` or reference which is + // permitted to read or modify the referent for the lifetime + // `'a`. Since we consume `self` by value, the returned pointer + // remains the only `Ptr` or reference which is permitted to + // read or modify the referent for the lifetime `'a`. + // - `Shared`: Since `self` has aliasing `Shared`, we know that + // no other code may mutate the referent during the lifetime + // `'a`, except via `UnsafeCell`s, and except as permitted by + // `T`'s library safety invariants. The caller promises that + // any safe operations which can be permitted on a `&T` and a + // `&U` simultaneously must be sound. Thus, no operations on a + // `&U` could violate `&T`'s library safety invariants, and + // vice-versa. Since any mutation via shared references outside + // of `UnsafeCell`s is unsound, this must be impossible using + // `&T` and `&U`. + // - `Inaccessible`: There are no restrictions we need to uphold. + // 1. `ptr` trivially satisfies the alignment invariant `Unaligned`. + // 2. The caller promises that the returned pointer satisfies the + // validity invariant `V` with respect to its referent type, `U`. + unsafe { Ptr::from_inner(ptr) } + } + } + + /// `Ptr<'a, T, (_, _, _)>` → `Ptr<'a, Unalign, (_, Aligned, _)>` + impl<'a, T, I> Ptr<'a, T, I> + where + I: Invariants, + { + /// Converts a `Ptr` an unaligned `T` into a `Ptr` to an aligned + /// `Unalign`. + #[inline] + #[must_use] + pub fn into_unalign( + self, + ) -> Ptr<'a, crate::Unalign, (I::Aliasing, Aligned, I::Validity)> { + // FIXME(#1359): This should be a `transmute_with` call. + // Unfortunately, to avoid blanket impl conflicts, we only implement + // `TransmuteFrom` for `Unalign` (and vice versa) specifically + // for `Valid` validity, not for all validity types. + + // SAFETY: + // - By `CastSized: Cast`, `CastSized` preserves referent address, + // and so we don't need to consider projections in the following + // safety arguments. + // - Since `Unalign` has the same layout as `T`, the returned + // pointer refers to `UnsafeCell`s at the same locations as + // `self`. + // - `Unalign` promises to have the same bit validity as `T`. By + // invariant on `Validity`, the set of bit patterns allowed in the + // referent of a `Ptr` is only a function of the + // validity of `X` and of `V`. Thus, the set of bit patterns + // allowed in the referent of a `Ptr` is + // the same as the set of bit patterns allowed in the referent of + // a `Ptr, (_, _, I::Validity)>`. As a result, `self` + // and the returned `Ptr` permit the same set of bit patterns in + // their referents, and so neither can be used to violate the + // validity of the other. + let ptr = unsafe { self.project_transmute_unchecked::<_, _, CastSized>() }; + ptr.bikeshed_recall_aligned() + } + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: ?Sized, + I: Invariants, + I::Aliasing: Reference, + { + /// Reads the referent. + #[must_use] + #[inline(always)] + pub fn read(self) -> T + where + T: Copy, + T: Read, + { + ::read(self) + } + + /// Views the value as an aligned reference. + /// + /// This is only available if `T` is [`Unaligned`]. + #[must_use] + #[inline] + pub fn unaligned_as_ref(self) -> &'a T + where + T: crate::Unaligned, + { + self.bikeshed_recall_aligned().as_ref() + } + } +} + +/// State transitions between invariants. +mod _transitions { + use super::*; + use crate::{ + pointer::{cast::IdCast, transmute::TryTransmuteFromPtr}, + ReadOnly, + }; + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Assumes that `self` satisfies the invariants `H`. + /// + /// # Safety + /// + /// The caller promises that `self` satisfies the invariants `H`. + unsafe fn assume_invariants(self) -> Ptr<'a, T, H> { + // SAFETY: The caller has promised to satisfy all parameterized + // invariants of `Ptr`. `Ptr`'s other invariants are satisfied + // by-contract by the source `Ptr`. + unsafe { Ptr::from_inner(self.as_inner()) } + } + + /// Helps the type system unify two distinct invariant types which are + /// actually the same. + #[inline] + #[must_use] + pub fn unify_invariants< + H: Invariants, + >( + self, + ) -> Ptr<'a, T, H> { + // SAFETY: The associated type bounds on `H` ensure that the + // invariants are unchanged. + unsafe { self.assume_invariants::() } + } + + /// Assumes that `self`'s referent is validly-aligned for `T` if + /// required by `A`. + /// + /// # Safety + /// + /// The caller promises that `self`'s referent conforms to the alignment + /// invariant of `T` if required by `A`. + #[inline] + pub(crate) unsafe fn assume_alignment( + self, + ) -> Ptr<'a, T, (I::Aliasing, A, I::Validity)> { + // SAFETY: The caller promises that `self`'s referent is + // well-aligned for `T` if required by `A` . + unsafe { self.assume_invariants() } + } + + /// Checks the `self`'s alignment at runtime, returning an aligned `Ptr` + /// on success. + #[inline] + pub fn try_into_aligned( + self, + ) -> Result, AlignmentError> + where + T: Sized, + { + if let Err(err) = + crate::util::validate_aligned_to::<_, T>(self.as_inner().as_non_null()) + { + return Err(err.with_src(self)); + } + + // SAFETY: We just checked the alignment. + Ok(unsafe { self.assume_alignment::() }) + } + + /// Recalls that `self`'s referent is validly-aligned for `T`. + #[inline] + // FIXME(#859): Reconsider the name of this method before making it + // public. + #[must_use] + pub fn bikeshed_recall_aligned(self) -> Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)> + where + T: crate::Unaligned, + { + // SAFETY: The bound `T: Unaligned` ensures that `T` has no + // non-trivial alignment requirement. + unsafe { self.assume_alignment::() } + } + + /// Assumes that `self`'s referent conforms to the validity requirement + /// of `V`. + /// + /// # Safety + /// + /// The caller promises that `self`'s referent conforms to the validity + /// requirement of `V`. + #[must_use] + #[inline] + pub unsafe fn assume_validity( + self, + ) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> { + // SAFETY: The caller promises that `self`'s referent conforms to + // the validity requirement of `V`. + unsafe { self.assume_invariants() } + } + + /// A shorthand for `self.assume_validity()`. + /// + /// # Safety + /// + /// The caller promises to uphold the safety preconditions of + /// `self.assume_validity()`. + #[must_use] + #[inline] + pub unsafe fn assume_initialized( + self, + ) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Initialized)> { + // SAFETY: The caller has promised to uphold the safety + // preconditions. + unsafe { self.assume_validity::() } + } + + /// A shorthand for `self.assume_validity()`. + /// + /// # Safety + /// + /// The caller promises to uphold the safety preconditions of + /// `self.assume_validity()`. + #[must_use] + #[inline] + pub unsafe fn assume_valid(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)> { + // SAFETY: The caller has promised to uphold the safety + // preconditions. + unsafe { self.assume_validity::() } + } + + /// Checks that `self`'s referent is validly initialized for `T`, + /// returning a `Ptr` with `Valid` on success. + /// + /// # Panics + /// + /// This method will panic if + /// [`T::is_bit_valid`][TryFromBytes::is_bit_valid] panics. + /// + /// # Safety + /// + /// On error, unsafe code may rely on this method's returned + /// `ValidityError` containing `self`. + #[inline] + pub fn try_into_valid( + mut self, + ) -> Result, ValidityError> + where + T: TryFromBytes + + Read + + TryTransmuteFromPtr, + ReadOnly: Read, + I::Aliasing: Reference, + I: Invariants, + { + // This call may panic. If that happens, it doesn't cause any + // soundness issues, as we have not generated any invalid state + // which we need to fix before returning. + if T::is_bit_valid(self.reborrow().transmute::<_, _, _>().reborrow_shared()) { + // SAFETY: If `T::is_bit_valid`, code may assume that `self` + // contains a bit-valid instance of `T`. By `T: + // TryTransmuteFromPtr`, so + // long as `self`'s referent conforms to the `Valid` validity + // for `T` (which we just confirmed), then this transmute is + // sound. + Ok(unsafe { self.assume_valid() }) + } else { + Err(ValidityError::new(self)) + } + } + + /// Forgets that `self`'s referent is validly-aligned for `T`. + #[inline] + #[must_use] + pub fn forget_aligned(self) -> Ptr<'a, T, (I::Aliasing, Unaligned, I::Validity)> { + // SAFETY: `Unaligned` is less restrictive than `Aligned`. + unsafe { self.assume_invariants() } + } + } +} + +/// Casts of the referent type. +#[cfg_attr(not(zerocopy_unstable_ptr), allow(unreachable_pub))] +pub use _casts::TryWithError; +mod _casts { + use core::cell::UnsafeCell; + + use super::*; + use crate::{ + pointer::cast::{AsBytesCast, Cast}, + HasTag, ProjectField, + }; + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Casts to a different referent type without checking interior + /// mutability. + /// + /// Callers should prefer [`cast`][Ptr::cast] where possible. + /// + /// # Safety + /// + /// If `I::Aliasing` is [`Shared`], it must not be possible for safe + /// code, operating on a `&T` and `&U` with the same referent + /// simultaneously, to cause undefined behavior. + #[inline] + #[must_use] + pub unsafe fn cast_unchecked>( + self, + ) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> + where + U: 'a + CastableFrom + ?Sized, + { + // SAFETY: + // - By `C: Cast`, `C` preserves the address of the referent. + // - If `I::Aliasing` is [`Shared`], the caller promises that it + // is not possible for safe code, operating on a `&T` and `&U` + // with the same referent simultaneously, to cause undefined + // behavior. + // - By `U: CastableFrom`, + // `I::Validity` is either `Uninit` or `Initialized`. In both + // cases, the bit validity `I::Validity` has the same semantics + // regardless of referent type. In other words, the set of allowed + // referent values for `Ptr` and `Ptr` are identical. As a consequence, neither + // `self` nor the returned `Ptr` can be used to write values which + // are invalid for the other. + unsafe { self.project_transmute_unchecked::<_, _, C>() } + } + + /// Casts to a different referent type. + #[inline] + #[must_use] + pub fn cast(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> + where + T: MutationCompatible, + U: 'a + ?Sized + CastableFrom, + C: Cast, + { + // SAFETY: Because `T: MutationCompatible`, one + // of the following holds: + // - `T: Read` and `U: Read`, in which + // case one of the following holds: + // - `I::Aliasing` is `Exclusive` + // - `T` and `U` are both `Immutable` + // - It is sound for safe code to operate on `&T` and `&U` with the + // same referent simultaneously. + unsafe { self.cast_unchecked::<_, C>() } + } + + #[inline(always)] + pub fn project( + mut self, + ) -> Result, T::Error> + where + T: ProjectField, + I::Aliasing: Reference, + { + use crate::pointer::cast::Projection; + match T::is_projectable(self.reborrow().project_tag()) { + Ok(()) => { + let inner = self.as_inner(); + let projected = inner.project::<_, Projection>(); + // SAFETY: By `T: ProjectField`, + // for `self: Ptr<'_, T, I>` such that `T::is_projectable` + // (which we've verified in this match arm), + // `T::project(self.as_inner())` conforms to + // `T::Invariants`. The `projected` pointer satisfies these + // invariants because it is produced by way of an + // abstraction that is equivalent to + // `T::project(ptr.as_inner())`: by invariant on + // `PtrInner::project`, `projected` is guaranteed to address + // the subset of the bytes of `inner`'s referent addressed + // by `Projection::project(inner)`, and by invariant on + // `Projection`, `Projection::project` is implemented by + // delegating to an implementation of `HasField::project`. + Ok(unsafe { Ptr::from_inner(projected) }) + } + Err(err) => Err(err), + } + } + + #[must_use] + #[inline(always)] + pub(crate) fn project_tag(self) -> Ptr<'a, T::Tag, I> + where + T: HasTag, + { + // SAFETY: By invariant on `Self::ProjectToTag`, this is a sound + // projection. + let tag = unsafe { self.project_transmute_unchecked::<_, _, T::ProjectToTag>() }; + // SAFETY: By invariant on `Self::ProjectToTag`, the projected + // pointer has the same alignment as `ptr`. + let tag = unsafe { tag.assume_alignment() }; + tag.unify_invariants() + } + + /// Attempts to transform the pointer, restoring the original on + /// failure. + /// + /// # Safety + /// + /// If `I::Aliasing != Shared`, then if `f` returns `Err(err)`, no copy + /// of `f`'s argument must exist outside of `err`. + #[inline(always)] + pub(crate) unsafe fn try_with_unchecked( + self, + f: F, + ) -> Result, E::Mapped> + where + U: 'a + ?Sized, + J: Invariants, + E: TryWithError, + F: FnOnce(Ptr<'a, T, I>) -> Result, E>, + { + let old_inner = self.as_inner(); + #[rustfmt::skip] + let res = f(self).map_err(#[inline(always)] move |err: E| { + err.map(#[inline(always)] |src| { + drop(src); + + // SAFETY: + // 0. Aliasing is either `Shared` or `Exclusive`: + // - If aliasing is `Shared`, then it cannot violate + // aliasing make another copy of this pointer (in fact, + // using `I::Aliasing = Shared`, we could have just + // cloned `self`). + // - If aliasing is `Exclusive`, then `f` is not allowed + // to make another copy of `self`. In `map_err`, we are + // consuming the only value in the returned `Result`. + // By invariant on `E: TryWithError`, that `err: + // E` only contains a single `Self` and no other + // non-ZST fields which could be `Ptr`s or references + // to `self`'s referent. By the same invariant, `map` + // consumes this single `Self` and passes it to this + // closure. Since `self` was, by invariant on + // `Exclusive`, the only `Ptr` or reference live for + // `'a` with this referent, and since we `drop(src)` + // above, there are no copies left, and so we are + // creating the only copy. + // 1. `self` conforms to `I::Aliasing` by invariant on + // `Ptr`, and `old_inner` has the same address, so it + // does too. + // 2. `f` could not have violated `self`'s validity without + // itself being unsound. Assuming that `f` is sound, the + // referent of `self` is still valid for `T`. + unsafe { Ptr::from_inner(old_inner) } + }) + }); + res + } + + /// Attempts to transform the pointer, restoring the original on + /// failure. + #[inline(always)] + pub fn try_with(self, f: F) -> Result, E::Mapped> + where + U: 'a + ?Sized, + J: Invariants, + E: TryWithError, + F: FnOnce(Ptr<'a, T, I>) -> Result, E>, + I: Invariants, + { + // SAFETY: `I::Aliasing = Shared`, so the safety condition does not + // apply. + unsafe { self.try_with_unchecked(f) } + } + } + + /// # Safety + /// + /// `Self` only contains a single `Self::Inner`, and `Self::Mapped` only + /// contains a single `MappedInner`. Other than that, `Self` and + /// `Self::Mapped` contain no non-ZST fields. + /// + /// `map` must pass ownership of `self`'s sole `Self::Inner` to `f`. + pub unsafe trait TryWithError { + type Inner; + type Mapped; + fn map MappedInner>(self, f: F) -> Self::Mapped; + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + KnownLayout + ?Sized, + I: Invariants, + { + /// Casts this pointer-to-initialized into a pointer-to-bytes. + #[allow(clippy::wrong_self_convention)] + #[must_use] + #[inline] + pub fn as_bytes(self) -> Ptr<'a, [u8], (I::Aliasing, Aligned, Valid)> + where + [u8]: TransmuteFromPtr, + { + self.transmute_with::<[u8], Valid, AsBytesCast, _>().bikeshed_recall_aligned() + } + } + + impl<'a, T, I, const N: usize> Ptr<'a, [T; N], I> + where + T: 'a, + I: Invariants, + { + /// Casts this pointer-to-array into a slice. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_slice(self) -> Ptr<'a, [T], I> { + let slice = self.as_inner().as_slice(); + // SAFETY: Note that, by post-condition on `PtrInner::as_slice`, + // `slice` refers to the same byte range as `self.as_inner()`. + // + // 0. Thus, `slice` conforms to the aliasing invariant of + // `I::Aliasing` because `self` does. + // 1. By the above lemma, `slice` conforms to the alignment + // invariant of `I::Alignment` because `self` does. + // 2. Since `[T; N]` and `[T]` have the same bit validity [1][2], + // and since `self` and the returned `Ptr` have the same validity + // invariant, neither `self` nor the returned `Ptr` can be used + // to write a value to the referent which violates the other's + // validity invariant. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout: + // + // An array of `[T; N]` has a size of `size_of::() * N` and the + // same alignment of `T`. Arrays are laid out so that the + // zero-based `nth` element of the array is offset from the start + // of the array by `n * size_of::()` bytes. + // + // ... + // + // Slices have the same layout as the section of the array they + // slice. + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/types/array.html#array-types: + // + // All elements of arrays are always initialized + unsafe { Ptr::from_inner(slice) } + } + } + + /// For caller convenience, these methods are generic over alignment + /// invariant. In practice, the referent is always well-aligned, because the + /// alignment of `[u8]` is 1. + impl<'a, I> Ptr<'a, [u8], I> + where + I: Invariants, + { + /// Attempts to cast `self` to a `U` using the given cast type. + /// + /// If `U` is a slice DST and pointer metadata (`meta`) is provided, + /// then the cast will only succeed if it would produce an object with + /// the given metadata. + /// + /// Returns `None` if the resulting `U` would be invalidly-aligned, if + /// no `U` can fit in `self`, or if the provided pointer metadata + /// describes an invalid instance of `U`. On success, returns a pointer + /// to the largest-possible `U` which fits in `self`. + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may + /// rely on that assumption for the soundness of their code. In + /// particular, the caller may assume that, if `try_cast_into` returns + /// `Some((ptr, remainder))`, then `ptr` and `remainder` refer to + /// non-overlapping byte ranges within `self`, and that `ptr` and + /// `remainder` entirely cover `self`. Finally: + /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a suffix cast, `remainder` has the same address as + /// `self`. + #[inline(always)] + pub fn try_cast_into( + self, + cast_type: CastType, + meta: Option, + ) -> Result< + (Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, Ptr<'a, [u8], I>), + CastError, + > + where + I::Aliasing: Reference, + U: 'a + ?Sized + KnownLayout + Read, + { + let (inner, remainder) = self.as_inner().try_cast_into(cast_type, meta).map_err( + #[inline(always)] + |err| { + err.map_src( + #[inline(always)] + |inner| + // SAFETY: `PtrInner::try_cast_into` promises to return its + // original argument on error, which was originally produced + // by `self.as_inner()`, which is guaranteed to satisfy + // `Ptr`'s invariants. + unsafe { Ptr::from_inner(inner) }, + ) + }, + )?; + + // SAFETY: + // 0. Since `U: Read`, either: + // - `I::Aliasing` is `Exclusive`, in which case both `src` and + // `ptr` conform to `Exclusive` + // - `I::Aliasing` is `Shared` and `U` is `Immutable` (we already + // know that `[u8]: Immutable`). In this case, neither `U` nor + // `[u8]` permit mutation, and so `Shared` aliasing is + // satisfied. + // 1. `ptr` conforms to the alignment invariant of `Aligned` because + // it is derived from `try_cast_into`, which promises that the + // object described by `target` is validly aligned for `U`. + // 2. By trait bound, `self` - and thus `target` - is a bit-valid + // `[u8]`. `Ptr<[u8], (_, _, Valid)>` and `Ptr<_, (_, _, + // Initialized)>` have the same bit validity, and so neither + // `self` nor `res` can be used to write a value to the referent + // which violates the other's validity invariant. + let res = unsafe { Ptr::from_inner(inner) }; + + // SAFETY: + // 0. `self` and `remainder` both have the type `[u8]`. Thus, they + // have `UnsafeCell`s at the same locations. Type casting does + // not affect aliasing. + // 1. `[u8]` has no alignment requirement. + // 2. `self` has validity `Valid` and has type `[u8]`. Since + // `remainder` references a subset of `self`'s referent, it is + // also a bit-valid `[u8]`. Thus, neither `self` nor `remainder` + // can be used to write a value to the referent which violates + // the other's validity invariant. + let remainder = unsafe { Ptr::from_inner(remainder) }; + + Ok((res, remainder)) + } + + /// Attempts to cast `self` into a `U`, failing if all of the bytes of + /// `self` cannot be treated as a `U`. + /// + /// In particular, this method fails if `self` is not validly-aligned + /// for `U` or if `self`'s size is not a valid size for `U`. + /// + /// # Safety + /// + /// On success, the caller may assume that the returned pointer + /// references the same byte range as `self`. + #[allow(unused)] + #[inline(always)] + pub fn try_cast_into_no_leftover( + self, + meta: Option, + ) -> Result, CastError> + where + I::Aliasing: Reference, + U: 'a + ?Sized + KnownLayout + Read, + [u8]: Read, + { + // SAFETY: The provided closure returns the only copy of `slf`. + unsafe { + self.try_with_unchecked( + #[inline(always)] + |slf| match slf.try_cast_into(CastType::Prefix, meta) { + Ok((slf, remainder)) => { + if remainder.is_empty() { + Ok(slf) + } else { + Err(CastError::Size(SizeError::<_, U>::new(()))) + } + } + Err(err) => Err(err.map_src( + #[inline(always)] + |_slf| (), + )), + }, + ) + } + } + } + + impl<'a, T, I> Ptr<'a, UnsafeCell, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Converts this `Ptr` into a pointer to the underlying data. + /// + /// This call borrows the `UnsafeCell` mutably (at compile-time) which + /// guarantees that we possess the only reference. + /// + /// This is like [`UnsafeCell::get_mut`], but for `Ptr`. + /// + /// [`UnsafeCell::get_mut`]: core::cell::UnsafeCell::get_mut + #[must_use] + #[inline(always)] + pub fn get_mut(self) -> Ptr<'a, T, I> { + // SAFETY: As described below, `UnsafeCell` has the same size + // as `T: ?Sized` (same static size or same DST layout). Thus, + // `*const UnsafeCell as *const T` is a size-preserving cast. + define_cast!(unsafe { Cast = UnsafeCell => T }); + + // SAFETY: + // - Aliasing is `Exclusive`, and so we are not required to promise + // anything about the locations of `UnsafeCell`s. + // - `UnsafeCell` has the same bit validity as `T` [1]. + // Technically the term "representation" doesn't guarantee this, + // but the subsequent sentence in the documentation makes it clear + // that this is the intention. + // + // By invariant on `Validity`, since `T` and `UnsafeCell` have + // the same bit validity, then the set of values which may appear + // in the referent of a `Ptr` is the same as the set + // which may appear in the referent of a `Ptr, (_, + // _, V)>`. Thus, neither `self` nor `ptr` may be used to write a + // value to the referent which would violate the other's validity + // invariant. + // + // [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell` has the same in-memory representation as its + // inner type `T`. A consequence of this guarantee is that it is + // possible to convert between `T` and `UnsafeCell`. + let ptr = unsafe { self.project_transmute_unchecked::<_, _, Cast>() }; + + // SAFETY: `UnsafeCell` has the same alignment as `T` [1], + // and so if `self` is guaranteed to be aligned, then so is the + // returned `Ptr`. + // + // [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell` has the same in-memory representation as + // its inner type `T`. A consequence of this guarantee is that + // it is possible to convert between `T` and `UnsafeCell`. + let ptr = unsafe { ptr.assume_alignment::() }; + ptr.unify_invariants() + } + } +} + +/// Projections through the referent. +mod _project { + use super::*; + + impl<'a, T, I> Ptr<'a, [T], I> + where + T: 'a, + I: Invariants, + I::Aliasing: Reference, + { + /// Iteratively projects the elements `Ptr` from `Ptr<[T]>`. + #[inline] + pub fn iter(self) -> impl Iterator> { + // SAFETY: + // 0. `elem` conforms to the aliasing invariant of `I::Aliasing`: + // - `Exclusive`: `self` is consumed by value, and therefore + // cannot be used to access the slice while any yielded + // element `Ptr` is live. Each non-zero-sized element is a + // disjoint byte range within the slice, and zero-sized + // elements address no bytes, so distinct yielded element + // `Ptr`s do not alias each other. + // - `Shared`: It is sound for multiple shared `Ptr`s to exist + // simultaneously which reference the same memory. + // 1. `elem`, conditionally, conforms to the validity invariant of + // `I::Alignment`. If `elem` is projected from data well-aligned + // for `[T]`, `elem` will be valid for `T`. + // 2. `elem` conforms to the validity invariant of `I::Validity`. + // Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout: + // + // Slices have the same layout as the section of the array they + // slice. + // + // Arrays are laid out so that the zero-based `nth` element of + // the array is offset from the start of the array by `n * + // size_of::()` bytes. Thus, `elem` addresses a valid `T` + // within the slice. Since `self` satisfies `I::Validity`, `elem` + // also satisfies `I::Validity`. + self.as_inner().iter().map( + #[inline(always)] + |elem| unsafe { Ptr::from_inner(elem) }, + ) + } + } + + #[allow(clippy::needless_lifetimes)] + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized + KnownLayout, + I: Invariants, + { + /// The number of slice elements in the object referenced by `self`. + #[inline] + #[must_use] + pub fn len(&self) -> usize { + self.as_inner().meta().get() + } + + /// Returns `true` if the slice pointer has a length of 0. + #[inline] + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + } +} + +#[cfg(test)] +mod tests { + use core::mem::{self, MaybeUninit}; + + use super::*; + #[allow(unused)] // Needed on our MSRV, but considered unused on later toolchains. + use crate::util::AsAddress; + use crate::{pointer::BecauseImmutable, util::testutil::AU64, FromBytes, Immutable}; + + mod test_ptr_try_cast_into_soundness { + use super::*; + + // This test is designed so that if `Ptr::try_cast_into_xxx` are + // buggy, it will manifest as unsoundness that Miri can detect. + + // - If `size_of::() == 0`, `N == 4` + // - Else, `N == 4 * size_of::()` + // + // Each test will be run for each metadata in `metas`. + fn test(metas: I) + where + T: ?Sized + KnownLayout + Immutable + FromBytes, + I: IntoIterator> + Clone, + { + let mut bytes = [MaybeUninit::::uninit(); N]; + let initialized = [MaybeUninit::new(0u8); N]; + for start in 0..=bytes.len() { + for end in start..=bytes.len() { + // Set all bytes to uninitialized other than those in + // the range we're going to pass to `try_cast_from`. + // This allows Miri to detect out-of-bounds reads + // because they read uninitialized memory. Without this, + // some out-of-bounds reads would still be in-bounds of + // `bytes`, and so might spuriously be accepted. + bytes = [MaybeUninit::::uninit(); N]; + let bytes = &mut bytes[start..end]; + // Initialize only the byte range we're going to pass to + // `try_cast_from`. + bytes.copy_from_slice(&initialized[start..end]); + + let bytes = { + let bytes: *const [MaybeUninit] = bytes; + #[allow(clippy::as_conversions)] + let bytes = bytes as *const [u8]; + // SAFETY: We just initialized these bytes to valid + // `u8`s. + unsafe { &*bytes } + }; + + // SAFETY: The bytes in `slf` must be initialized. + unsafe fn validate_and_get_len< + T: ?Sized + KnownLayout + FromBytes + Immutable, + >( + slf: Ptr<'_, T, (Shared, Aligned, Initialized)>, + ) -> usize { + let t = slf.recall_validity().as_ref(); + + let bytes = { + let len = mem::size_of_val(t); + let t: *const T = t; + // SAFETY: + // - We know `t`'s bytes are all initialized + // because we just read it from `slf`, which + // points to an initialized range of bytes. If + // there's a bug and this doesn't hold, then + // that's exactly what we're hoping Miri will + // catch! + // - Since `T: FromBytes`, `T` doesn't contain + // any `UnsafeCell`s, so it's okay for `t: T` + // and a `&[u8]` to the same memory to be + // alive concurrently. + unsafe { core::slice::from_raw_parts(t.cast::(), len) } + }; + + // This assertion ensures that `t`'s bytes are read + // and compared to another value, which in turn + // ensures that Miri gets a chance to notice if any + // of `t`'s bytes are uninitialized, which they + // shouldn't be (see the comment above). + assert_eq!(bytes, vec![0u8; bytes.len()]); + + mem::size_of_val(t) + } + + for meta in metas.clone().into_iter() { + for cast_type in [CastType::Prefix, CastType::Suffix] { + if let Ok((slf, remaining)) = Ptr::from_ref(bytes) + .try_cast_into::(cast_type, meta) + { + // SAFETY: All bytes in `bytes` have been + // initialized. + let len = unsafe { validate_and_get_len(slf) }; + assert_eq!(remaining.len(), bytes.len() - len); + #[allow(unstable_name_collisions)] + let bytes_addr = bytes.as_ptr().addr(); + #[allow(unstable_name_collisions)] + let remaining_addr = remaining.as_inner().as_ptr().addr(); + match cast_type { + CastType::Prefix => { + assert_eq!(remaining_addr, bytes_addr + len) + } + CastType::Suffix => assert_eq!(remaining_addr, bytes_addr), + } + + if let Some(want) = meta { + let got = + KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); + assert_eq!(got, want); + } + } + } + + if let Ok(slf) = Ptr::from_ref(bytes) + .try_cast_into_no_leftover::(meta) + { + // SAFETY: All bytes in `bytes` have been + // initialized. + let len = unsafe { validate_and_get_len(slf) }; + assert_eq!(len, bytes.len()); + + if let Some(want) = meta { + let got = KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); + assert_eq!(got, want); + } + } + } + } + } + } + + #[derive(FromBytes, KnownLayout, Immutable)] + #[repr(C)] + struct SliceDst { + a: u8, + trailing: [T], + } + + // Each test case becomes its own `#[test]` function. We do this because + // this test in particular takes far, far longer to execute under Miri + // than all of our other tests combined. Previously, we had these + // execute sequentially in a single test function. We run Miri tests in + // parallel in CI, but this test being sequential meant that most of + // that parallelism was wasted, as all other tests would finish in a + // fraction of the total execution time, leaving this test to execute on + // a single thread for the remainder of the test. By putting each test + // case in its own function, we permit better use of available + // parallelism. + macro_rules! test { + ($test_name:ident: $ty:ty) => { + #[test] + #[allow(non_snake_case)] + fn $test_name() { + const S: usize = core::mem::size_of::<$ty>(); + const N: usize = if S == 0 { 4 } else { S * 4 }; + test::<$ty, _, N>([None]); + + // If `$ty` is a ZST, then we can't pass `None` as the + // pointer metadata, or else computing the correct trailing + // slice length will panic. + if S == 0 { + test::<[$ty], _, N>([Some(0), Some(1), Some(2), Some(3)]); + test::, _, N>([Some(0), Some(1), Some(2), Some(3)]); + } else { + test::<[$ty], _, N>([None, Some(0), Some(1), Some(2), Some(3)]); + test::, _, N>([None, Some(0), Some(1), Some(2), Some(3)]); + } + } + }; + ($ty:ident) => { + test!($ty: $ty); + }; + ($($ty:ident),*) => { $(test!($ty);)* } + } + + test!(empty_tuple: ()); + test!(u8, u16, u32, u64, usize, AU64); + test!(i8, i16, i32, i64, isize); + test!(f32, f64); + } + + #[test] + fn test_try_cast_into_explicit_count() { + macro_rules! test { + ($ty:ty, $bytes:expr, $elems:expr, $expect:expr) => {{ + let bytes = [0u8; $bytes]; + let ptr = Ptr::from_ref(&bytes[..]); + let res = + ptr.try_cast_into::<$ty, BecauseImmutable>(CastType::Prefix, Some($elems)); + if let Some(expect) = $expect { + let (ptr, _) = res.unwrap(); + assert_eq!(KnownLayout::pointer_to_metadata(ptr.as_inner().as_ptr()), expect); + } else { + let _ = res.unwrap_err(); + } + }}; + } + + #[derive(KnownLayout, Immutable)] + #[repr(C)] + struct ZstDst { + u: [u8; 8], + slc: [()], + } + + test!(ZstDst, 8, 0, Some(0)); + test!(ZstDst, 7, 0, None); + + test!(ZstDst, 8, usize::MAX, Some(usize::MAX)); + test!(ZstDst, 7, usize::MAX, None); + + #[derive(KnownLayout, Immutable)] + #[repr(C)] + struct Dst { + u: [u8; 8], + slc: [u8], + } + + test!(Dst, 8, 0, Some(0)); + test!(Dst, 7, 0, None); + + test!(Dst, 9, 1, Some(1)); + test!(Dst, 8, 1, None); + + // If we didn't properly check for overflow, this would cause the + // metadata to overflow to 0, and thus the cast would spuriously + // succeed. + test!(Dst, 8, usize::MAX - 8 + 1, None); + } + + #[test] + fn test_try_cast_into_no_leftover_restores_original_slice() { + let bytes = [0u8; 4]; + let ptr = Ptr::from_ref(&bytes[..]); + let res = ptr.try_cast_into_no_leftover::<[u8; 2], BecauseImmutable>(None); + match res { + Ok(_) => panic!("should have failed due to leftover bytes"), + Err(CastError::Size(e)) => { + assert_eq!(e.into_src().len(), 4, "Should return original slice length"); + } + Err(e) => panic!("wrong error type: {:?}", e), + } + } + + #[test] + fn test_iter_exclusive_yields_disjoint_ptrs() { + let mut arr = [0u8, 1, 2, 3]; + + { + let mut iter = Ptr::from_mut(&mut arr[..]).iter(); + let first = iter.next().unwrap().as_mut(); + let second = iter.next().unwrap().as_mut(); + + *first = 10; + *second = 20; + *first = 30; + } + + assert_eq!(arr, [30, 20, 2, 3]); + } +} diff --git a/rust/zerocopy/src/pointer/transmute.rs b/rust/zerocopy/src/pointer/transmute.rs new file mode 100644 index 000000000000..45b4f727fd0d --- /dev/null +++ b/rust/zerocopy/src/pointer/transmute.rs @@ -0,0 +1,520 @@ +// Copyright 2025 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_docs)] + +use core::{ + cell::{Cell, UnsafeCell}, + mem::{ManuallyDrop, MaybeUninit}, + num::Wrapping, +}; + +use crate::{ + pointer::{ + cast::{self, CastExact, CastSizedExact}, + invariant::*, + }, + FromBytes, Immutable, IntoBytes, Unalign, +}; + +/// Transmutations which are sound to attempt, conditional on validating the bit +/// validity of the destination type. +/// +/// If a `Ptr` transmutation is `TryTransmuteFromPtr`, then it is sound to +/// perform that transmutation so long as some additional mechanism is used to +/// validate that the referent is bit-valid for the destination type. That +/// validation mechanism could be a type bound (such as `TransmuteFrom`) or a +/// runtime validity check. +/// +/// # Safety +/// +/// ## Post-conditions +/// +/// Given `Dst: TryTransmuteFromPtr`, callers may assume +/// the following: +/// +/// Given `src: Ptr<'a, Src, (A, _, SV)>`, if the referent of `src` is +/// `DV`-valid for `Dst`, then it is sound to transmute `src` into `dst: Ptr<'a, +/// Dst, (A, Unaligned, DV)>` using `C`. +/// +/// ## Pre-conditions +/// +/// Given `src: Ptr` and `dst: Ptr`, +/// `Dst: TryTransmuteFromPtr` is sound if all of the +/// following hold: +/// - Forwards transmutation: Either of the following hold: +/// - So long as `dst` is active, no mutation of `dst`'s referent is allowed +/// except via `dst` itself +/// - The set of `DV`-valid referents of `dst` is a superset of the set of +/// `SV`-valid referents of `src` (NOTE: this condition effectively bans +/// shrinking or overwriting transmutes, which cannot satisfy this +/// condition) +/// - Reverse transmutation: Either of the following hold: +/// - `dst` does not permit mutation of its referent +/// - The set of `DV`-valid referents of `dst` is a subset of the set of +/// `SV`-valid referents of `src` (NOTE: this condition effectively bans +/// shrinking or overwriting transmutes, which cannot satisfy this +/// condition) +/// - No safe code, given access to `src` and `dst`, can cause undefined +/// behavior: Any of the following hold: +/// - `A` is `Exclusive` +/// - `Src: Immutable` and `Dst: Immutable` +/// - It is sound for shared code to operate on a `&Src` and `&Dst` which +/// reference the same byte range at the same time +/// +/// ## Proof +/// +/// Given: +/// - `src: Ptr<'a, Src, (A, _, SV)>` +/// - `src`'s referent is `DV`-valid for `Dst` +/// +/// We are trying to prove that it is sound to perform a cast from `src` to a +/// `dst: Ptr<'a, Dst, (A, Unaligned, DV)>` using `C`. We need to prove that +/// such a cast does not violate any of `src`'s invariants, and that it +/// satisfies all invariants of the destination `Ptr` type. +/// +/// First, by `C: CastExact`, `src`'s address is unchanged, so it still satisfies +/// its alignment. Since `dst`'s alignment is `Unaligned`, it trivially satisfies +/// its alignment. +/// +/// Second, aliasing is either `Exclusive` or `Shared`: +/// - If it is `Exclusive`, then both `src` and `dst` satisfy `Exclusive` +/// aliasing trivially: since `src` and `dst` have the same lifetime, `src` is +/// inaccessible so long as `dst` is alive, and no other live `Ptr`s or +/// references may reference the same referent. +/// - If it is `Shared`, then either: +/// - `Src: Immutable` and `Dst: Immutable`, and so neither `src` nor `dst` +/// permit interior mutation. +/// - It is explicitly sound for safe code to operate on a `&Src` and a `&Dst` +/// pointing to the same byte range at the same time. +/// +/// Third, `src`'s validity is satisfied. By invariant, `src`'s referent began +/// as an `SV`-valid `Src`. It is guaranteed to remain so, as either of the +/// following hold: +/// - `dst` does not permit mutation of its referent. +/// - The set of `DV`-valid referents of `dst` is a subset of the set of +/// `SV`-valid referents of `src`. Thus, any value written via `dst` is +/// guaranteed to be an `SV`-valid referent of `src`. +/// +/// Fourth, `dst`'s validity is satisfied. It is a given of this proof that the +/// referent is `DV`-valid for `Dst`. It is guaranteed to remain so, as either +/// of the following hold: +/// - So long as `dst` is active, no mutation of the referent is allowed except +/// via `dst` itself. +/// - The set of `DV`-valid referents of `dst` is a superset of the set of +/// `SV`-valid referents of `src`. Thus, any value written via `src` is +/// guaranteed to be a `DV`-valid referent of `dst`. +pub unsafe trait TryTransmuteFromPtr< + Src: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact, + R, +> +{ +} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub enum BecauseMutationCompatible {} + +// SAFETY: +// - Forwards transmutation: By `Dst: MutationCompatible`, we +// know that at least one of the following holds: +// - So long as `dst: Ptr` is active, no mutation of its referent is +// allowed except via `dst` itself if either of the following hold: +// - Aliasing is `Exclusive`, in which case, so long as the `Dst` `Ptr` +// exists, no mutation is permitted except via that `Ptr` +// - Aliasing is `Shared`, `Src: Immutable`, and `Dst: Immutable`, in which +// case no mutation is possible via either `Ptr` +// - Since the underlying cast is size-preserving, `dst` addresses the same +// referent as `src`. By `Dst: TransmuteFrom`, the set of +// `DV`-valid referents of `dst` is a superset of the set of `SV`-valid +// referents of `src`. +// - Reverse transmutation: Since the underlying cast is size-preserving, `dst` +// addresses the same referent as `src`. By `Src: TransmuteFrom`, +// the set of `DV`-valid referents of `src` is a subset of the set of +// `SV`-valid referents of `dst`. +// - No safe code, given access to `src` and `dst`, can cause undefined +// behavior: By `Dst: MutationCompatible`, at least one of +// the following holds: +// - `A` is `Exclusive` +// - `Src: Immutable` and `Dst: Immutable` +// - `Dst: InvariantsEq`, which guarantees that `Src` and `Dst` have the +// same invariants, and permit interior mutation on the same byte ranges +unsafe impl + TryTransmuteFromPtr for Dst +where + A: Aliasing, + SV: Validity, + DV: Validity, + Src: TransmuteFrom + ?Sized, + Dst: MutationCompatible + ?Sized, + C: CastExact, +{ +} + +// SAFETY: +// - Forwards transmutation: Since aliasing is `Shared` and `Src: Immutable`, +// `src` does not permit mutation of its referent. +// - Reverse transmutation: Since aliasing is `Shared` and `Dst: Immutable`, +// `dst` does not permit mutation of its referent. +// - No safe code, given access to `src` and `dst`, can cause undefined +// behavior: `Src: Immutable` and `Dst: Immutable` +unsafe impl TryTransmuteFromPtr + for Dst +where + SV: Validity, + DV: Validity, + Src: Immutable + ?Sized, + Dst: Immutable + ?Sized, + C: CastExact, +{ +} + +/// Denotes that `src: Ptr` and `dst: Ptr`, +/// referencing the same referent at the same time, cannot be used by safe code +/// to break library safety invariants of `Src` or `Self`. +/// +/// # Safety +/// +/// At least one of the following must hold: +/// - `Src: Read` and `Self: Read` +/// - `Self: InvariantsEq`, and, for some `V`: +/// - `Dst: TransmuteFrom` +/// - `Src: TransmuteFrom` +pub unsafe trait MutationCompatible {} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub enum BecauseRead {} + +// SAFETY: `Src: Read` and `Dst: Read`. +unsafe impl + MutationCompatible for Dst +where + Src: Read, + Dst: Read, +{ +} + +/// Denotes that two types have the same invariants. +/// +/// # Safety +/// +/// It is sound for safe code to operate on a `&T` and a `&Self` pointing to the +/// same referent at the same time - no such safe code can cause undefined +/// behavior. +pub unsafe trait InvariantsEq {} + +// SAFETY: Trivially sound to have multiple `&T` pointing to the same referent. +unsafe impl InvariantsEq for T {} + +// SAFETY: `Dst: InvariantsEq + TransmuteFrom`, and `Src: +// TransmuteFrom`. +unsafe impl + MutationCompatible for Dst +where + Src: TransmuteFrom, + Dst: TransmuteFrom + InvariantsEq, +{ +} + +#[allow(missing_debug_implementations, missing_copy_implementations)] +pub enum BecauseInvariantsEq {} + +macro_rules! unsafe_impl_invariants_eq { + ($tyvar:ident => $t:ty, $u:ty) => {{ + crate::util::macros::__unsafe(); + // SAFETY: The caller promises that this is sound. + unsafe impl<$tyvar> InvariantsEq<$t> for $u {} + // SAFETY: The caller promises that this is sound. + unsafe impl<$tyvar> InvariantsEq<$u> for $t {} + }}; +} + +impl_transitive_transmute_from!(T => MaybeUninit => T => Wrapping); +impl_transitive_transmute_from!(T => Wrapping => T => MaybeUninit); + +// SAFETY: `ManuallyDrop` has the same size and bit validity as `T` [1], and +// implements `Deref` [2]. Thus, it is already possible for safe +// code to obtain a `&T` and a `&ManuallyDrop` to the same referent at the +// same time. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop` is guaranteed to have the same layout and bit +// validity as `T` +// +// [2] https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html#impl-Deref-for-ManuallyDrop%3CT%3E +unsafe impl InvariantsEq for ManuallyDrop {} +// SAFETY: See previous safety comment. +unsafe impl InvariantsEq> for T {} + +/// Transmutations which are always sound. +/// +/// `TransmuteFromPtr` is a shorthand for [`TryTransmuteFromPtr`] and +/// [`TransmuteFrom`]. +/// +/// # Safety +/// +/// `Dst: TransmuteFromPtr` is equivalent to `Dst: +/// TryTransmuteFromPtr + TransmuteFrom`. +pub unsafe trait TransmuteFromPtr< + Src: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact, + R, +>: TryTransmuteFromPtr + TransmuteFrom +{ +} + +// SAFETY: The `where` bounds are equivalent to the safety invariant on +// `TransmuteFromPtr`. +unsafe impl< + Src: ?Sized, + Dst: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact, + R, + > TransmuteFromPtr for Dst +where + Dst: TransmuteFrom + TryTransmuteFromPtr, +{ +} + +/// Denotes that any `SV`-valid `Src` may soundly be transmuted into a +/// `DV`-valid `Self`. +/// +/// # Safety +/// +/// Given `src: Ptr` and `dst: Ptr`, if the +/// referents of `src` and `dst` are the same size, then the set of bit patterns +/// allowed to appear in `src`'s referent must be a subset of the set allowed to +/// appear in `dst`'s referent. +/// +/// If the referents are not the same size, then `Dst: TransmuteFrom` conveys no safety guarantee. +pub unsafe trait TransmuteFrom {} + +/// Carries the ability to perform a size-preserving cast or conversion from a +/// raw pointer to `Src` to a raw pointer to `Self`. +/// +/// The cast/conversion is carried by the associated [`CastFrom`] type, and +/// may be a no-op cast (without updating pointer metadata) or a conversion +/// which updates pointer metadata. +/// +/// # Safety +/// +/// `SizeEq` on its own conveys no safety guarantee. Any safety guarantees come +/// from the safety invariants on the associated [`CastFrom`] type, specifically +/// the [`CastExact`] bound. +/// +/// [`CastFrom`]: SizeEq::CastFrom +/// [`CastExact`]: CastExact +pub trait SizeEq { + type CastFrom: CastExact; +} + +impl SizeEq for T { + type CastFrom = cast::IdCast; +} + +// SAFETY: Since `Src: IntoBytes`, the set of valid `Src`'s is the set of +// initialized bit patterns, which is exactly the set allowed in the referent of +// any `Initialized` `Ptr`. +unsafe impl TransmuteFrom for Dst +where + Src: IntoBytes + ?Sized, + Dst: ?Sized, +{ +} + +// SAFETY: Since `Dst: FromBytes`, any initialized bit pattern may appear in the +// referent of a `Ptr`. This is exactly equal to the set of +// bit patterns which may appear in the referent of any `Initialized` `Ptr`. +unsafe impl TransmuteFrom for Dst +where + Src: ?Sized, + Dst: FromBytes + ?Sized, +{ +} + +// FIXME(#2354): This seems like a smell - the soundness of this bound has +// nothing to do with `Src` or `Dst` - we're basically just saying `[u8; N]` is +// transmutable into `[u8; N]`. + +// SAFETY: The set of allowed bit patterns in the referent of any `Initialized` +// `Ptr` is the same regardless of referent type. +unsafe impl TransmuteFrom for Dst +where + Src: ?Sized, + Dst: ?Sized, +{ +} + +// FIXME(#2354): This seems like a smell - the soundness of this bound has +// nothing to do with `Dst` - we're basically just saying that any type is +// transmutable into `MaybeUninit<[u8; N]>`. + +// SAFETY: A `Dst` with validity `Uninit` permits any byte sequence, and +// therefore can be transmuted from any value. +unsafe impl TransmuteFrom for Dst +where + Src: ?Sized, + Dst: ?Sized, + V: Validity, +{ +} + +// SAFETY: +// - `ManuallyDrop` has the same size as `T` [1] +// - `ManuallyDrop` has the same validity as `T` [1] +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop` is guaranteed to have the same layout and bit validity as +// `T` +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => ManuallyDrop) }; + +// SAFETY: +// - `Unalign` promises to have the same size as `T`. +// - `Unalign` promises to have the same validity as `T`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T => Unalign) }; +// SAFETY: `Unalign` promises to have the same size and validity as `T`. +// Given `u: &Unalign`, it is already possible to obtain `let t = +// u.try_deref().unwrap()`. Because `Unalign` has the same size as `T`, the +// returned `&T` must point to the same referent as `u`, and thus it must be +// sound for these two references to exist at the same time since it's already +// possible for safe code to get into this state. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Unalign) }; + +// SAFETY: +// - `Wrapping` has the same size as `T` [1]. +// - `Wrapping` has only one field, which is `pub` [2]. We are also +// guaranteed per that `Wrapping` has the same layout as `T` [1]. The only +// way for both of these to be true simultaneously is for `Wrapping` to +// have the same bit validity as `T`. In particular, in order to change the +// bit validity, one of the following would need to happen: +// - `Wrapping` could change its `repr`, but this would violate the layout +// guarantee. +// - `Wrapping` could add or change its fields, but this would be a +// stability-breaking change. +// +// [1] Per https://doc.rust-lang.org/1.85.0/core/num/struct.Wrapping.html#layout-1: +// +// `Wrapping` is guaranteed to have the same layout and ABI as `T`. +// +// [2] Definition from https://doc.rust-lang.org/1.85.0/core/num/struct.Wrapping.html: +// +// ``` +// #[repr(transparent)] +// pub struct Wrapping(pub T); +// ``` +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T => Wrapping) }; + +// SAFETY: By the preceding safety proof, `Wrapping` and `T` have the same +// layout and bit validity. Since a `Wrapping`'s `T` field is `pub`, given +// `w: &Wrapping`, it's possible to do `let t = &w.t`, which means that it's +// already possible for safe code to obtain a `&Wrapping` and a `&T` pointing +// to the same referent at the same time. Thus, this must be sound. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Wrapping) }; + +// SAFETY: +// - `UnsafeCell` has the same size as `T` [1]. +// - Per [1], `UnsafeCell` has the same bit validity as `T`. Technically the +// term "representation" doesn't guarantee this, but the subsequent sentence +// in the documentation makes it clear that this is the intention. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell` has the same in-memory representation as its inner type +// `T`. A consequence of this guarantee is that it is possible to convert +// between `T` and `UnsafeCell`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => UnsafeCell) }; + +// SAFETY: +// - `Cell` has the same size as `T` [1]. +// - Per [1], `Cell` has the same bit validity as `T`. Technically the term +// "representation" doesn't guarantee this, but it does promise to have the +// "same memory layout and caveats as `UnsafeCell`." The `UnsafeCell` docs +// [2] make it clear that bit validity is the intention even if that phrase +// isn't used. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/cell/struct.Cell.html#memory-layout: +// +// `Cell` has the same memory layout and caveats as `UnsafeCell`. In +// particular, this means that `Cell` has the same in-memory representation +// as its inner type `T`. +// +// [2] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell` has the same in-memory representation as its inner type +// `T`. A consequence of this guarantee is that it is possible to convert +// between `T` and `UnsafeCell`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => Cell) }; + +impl_transitive_transmute_from!(T: ?Sized => Cell => T => UnsafeCell); +impl_transitive_transmute_from!(T: ?Sized => UnsafeCell => T => Cell); + +// SAFETY: `MaybeUninit` has no validity requirements. Currently this is not +// explicitly guaranteed, but it's obvious from `MaybeUninit`'s documentation +// that this is the intention: +// https://doc.rust-lang.org/1.85.0/core/mem/union.MaybeUninit.html +unsafe impl TransmuteFrom for MaybeUninit {} + +impl SizeEq for MaybeUninit { + type CastFrom = CastSizedExact; +} + +impl SizeEq> for T { + type CastFrom = CastSizedExact; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pointer::cast::Project as _; + + fn test_size_eq>(mut src: Src) { + let _: *mut Dst = + >::CastFrom::project(crate::pointer::PtrInner::from_mut(&mut src)); + } + + #[test] + fn test_transmute_coverage() { + // SizeEq for MaybeUninit + test_size_eq::>(0u8); + + // SizeEq> for T + test_size_eq::, u8>(MaybeUninit::::new(0)); + + // Transitive: MaybeUninit -> Wrapping + // T => MaybeUninit => T => Wrapping + test_size_eq::>(0u8); + + // T => Wrapping => T => MaybeUninit + test_size_eq::, MaybeUninit>(Wrapping(0u8)); + + // T: ?Sized => Cell => T => UnsafeCell + test_size_eq::, UnsafeCell>(Cell::new(0u8)); + + // T: ?Sized => UnsafeCell => T => Cell + test_size_eq::, Cell>(UnsafeCell::new(0u8)); + } +} diff --git a/rust/zerocopy/src/ref.rs b/rust/zerocopy/src/ref.rs new file mode 100644 index 000000000000..496afc435c61 --- /dev/null +++ b/rust/zerocopy/src/ref.rs @@ -0,0 +1,1356 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. +use super::*; +use crate::pointer::{ + BecauseInvariantsEq, BecauseMutationCompatible, MutationCompatible, TransmuteFromPtr, +}; + +mod def { + use core::marker::PhantomData; + + use crate::{ + ByteSlice, ByteSliceMut, CloneableByteSlice, CopyableByteSlice, IntoByteSlice, + IntoByteSliceMut, + }; + + /// A typed reference derived from a byte slice. + /// + /// A `Ref` is a reference to a `T` which is stored in a byte slice, `B`. + /// Unlike a native reference (`&T` or `&mut T`), `Ref` has the same + /// mutability as the byte slice it was constructed from (`B`). + /// + /// # Examples + /// + /// `Ref` can be used to treat a sequence of bytes as a structured type, and + /// to read and write the fields of that type as if the byte slice reference + /// were simply a reference to that type. + /// + /// ```rust + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct UdpHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C, packed)] + /// struct UdpPacket { + /// header: UdpHeader, + /// body: [u8], + /// } + /// + /// impl UdpPacket { + /// pub fn parse(bytes: B) -> Option> { + /// Ref::from_bytes(bytes).ok() + /// } + /// } + /// ``` + pub struct Ref( + // INVARIANTS: The referent (via `.deref`, `.deref_mut`, `.into`) byte + // slice is aligned to `T`'s alignment and its size corresponds to a + // valid size for `T`. + B, + PhantomData, + ); + + impl Ref { + /// Constructs a new `Ref`. + /// + /// # Safety + /// + /// `bytes` dereferences (via [`deref`], [`deref_mut`], and [`into`]) to + /// a byte slice which is aligned to `T`'s alignment and whose size is a + /// valid size for `T`. + /// + /// [`deref`]: core::ops::Deref::deref + /// [`deref_mut`]: core::ops::DerefMut::deref_mut + /// [`into`]: core::convert::Into::into + pub(crate) unsafe fn new_unchecked(bytes: B) -> Ref { + // INVARIANTS: The caller has promised that `bytes`'s referent is + // validly-aligned and has a valid size. + Ref(bytes, PhantomData) + } + } + + impl Ref { + /// Access the byte slice as a [`ByteSlice`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`ByteSlice`] other than `ByteSlice` methods (for example, via + /// `Any::downcast_ref`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn as_byte_slice(&self) -> &impl ByteSlice { + // INVARIANTS: The caller promises not to call methods other than + // those on `ByteSlice`. Since `B: ByteSlice`, dereference stability + // guarantees that calling `ByteSlice` methods will not change the + // address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + &self.0 + } + } + + impl Ref { + /// Access the byte slice as a [`ByteSliceMut`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`ByteSliceMut`] other than `ByteSliceMut` methods (for example, via + /// `Any::downcast_mut`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn as_byte_slice_mut(&mut self) -> &mut impl ByteSliceMut { + // INVARIANTS: The caller promises not to call methods other than + // those on `ByteSliceMut`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + &mut self.0 + } + } + + impl<'a, B: IntoByteSlice<'a>, T: ?Sized> Ref { + /// Access the byte slice as an [`IntoByteSlice`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`IntoByteSlice`] other than `IntoByteSlice` methods (for example, + /// via `Any::downcast_ref`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn into_byte_slice(self) -> impl IntoByteSlice<'a> { + // INVARIANTS: The caller promises not to call methods other than + // those on `IntoByteSlice`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + self.0 + } + } + + impl<'a, B: IntoByteSliceMut<'a>, T: ?Sized> Ref { + /// Access the byte slice as an [`IntoByteSliceMut`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`IntoByteSliceMut`] other than `IntoByteSliceMut` methods (for + /// example, via `Any::downcast_mut`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn into_byte_slice_mut(self) -> impl IntoByteSliceMut<'a> { + // INVARIANTS: The caller promises not to call methods other than + // those on `IntoByteSliceMut`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + self.0 + } + } + + impl Clone for Ref { + #[inline] + fn clone(&self) -> Ref { + // INVARIANTS: Since `B: CloneableByteSlice`, `self.0.clone()` has + // the same address and length as `self.0`. Since `self.0` upholds + // the field invariants, so does `self.0.clone()`. + Ref(self.0.clone(), PhantomData) + } + } + + // INVARIANTS: Since `B: CopyableByteSlice`, the copied `Ref`'s `.0` has the + // same address and length as the original `Ref`'s `.0`. Since the original + // upholds the field invariants, so does the copy. + impl Copy for Ref {} +} + +#[allow(unreachable_pub)] // This is a false positive on our MSRV toolchain. +pub use def::Ref; + +use crate::pointer::{ + invariant::{Aligned, BecauseExclusive, Initialized, Unaligned, Valid}, + BecauseRead, PtrInner, +}; + +impl Ref +where + B: ByteSlice, +{ + #[must_use = "has no side effects"] + pub(crate) fn sized_from(bytes: B) -> Result, CastError> { + if bytes.len() != mem::size_of::() { + return Err(SizeError::new(bytes).into()); + } + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + + // SAFETY: We just validated size and alignment. + Ok(unsafe { Ref::new_unchecked(bytes) }) + } +} + +impl Ref +where + B: SplitByteSlice, +{ + #[must_use = "has no side effects"] + pub(crate) fn sized_from_prefix(bytes: B) -> Result<(Ref, B), CastError> { + if bytes.len() < mem::size_of::() { + return Err(SizeError::new(bytes).into()); + } + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + let (bytes, suffix) = bytes.split_at(mem::size_of::()).map_err( + #[inline(always)] + |b| SizeError::new(b).into(), + )?; + // SAFETY: We just validated alignment and that `bytes` is at least as + // large as `T`. `bytes.split_at(mem::size_of::())?` ensures that the + // new `bytes` is exactly the size of `T`. By safety postcondition on + // `SplitByteSlice::split_at` we can rely on `split_at` to produce the + // correct `bytes` and `suffix`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((r, suffix)) + } + + #[must_use = "has no side effects"] + pub(crate) fn sized_from_suffix(bytes: B) -> Result<(B, Ref), CastError> { + let bytes_len = bytes.len(); + let split_at = if let Some(split_at) = bytes_len.checked_sub(mem::size_of::()) { + split_at + } else { + return Err(SizeError::new(bytes).into()); + }; + let (prefix, bytes) = bytes.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + // SAFETY: Since `split_at` is defined as `bytes_len - size_of::()`, + // the `bytes` which results from `let (prefix, bytes) = + // bytes.split_at(split_at)?` has length `size_of::()`. After + // constructing `bytes`, we validate that it has the proper alignment. + // By safety postcondition on `SplitByteSlice::split_at` we can rely on + // `split_at` to produce the correct `prefix` and `bytes`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((prefix, r)) + } +} + +impl Ref +where + B: ByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + /// Constructs a `Ref` from a byte slice. + /// + /// If the length of `source` is not a [valid size of `T`][valid-size], or + /// if `source` is not appropriately aligned for `T`, this returns `Err`. If + /// [`T: Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_bytes(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_bytes(source: B) -> Result, CastError> { + static_assert_dst_is_not_zst!(T); + if let Err(e) = + Ptr::from_ref(source.deref()).try_cast_into_no_leftover::(None) + { + return Err(e.with_src(()).with_src(source)); + } + // SAFETY: `try_cast_into_no_leftover` validates size and alignment. + Ok(unsafe { Ref::new_unchecked(source) }) + } +} + +impl Ref +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the prefix of a byte slice. + /// + /// This method computes the [largest possible size of `T`][valid-size] that + /// can fit in the leading bytes of `source`, then attempts to return both a + /// `Ref` to those bytes, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, + /// this returns `Err`. If [`T: Unaligned`][t-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_prefix(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_prefix(source: B) -> Result<(Ref, B), CastError> { + static_assert_dst_is_not_zst!(T); + let remainder = match Ptr::from_ref(source.deref()) + .try_cast_into::(CastType::Prefix, None) + { + Ok((_, remainder)) => remainder, + Err(e) => { + return Err(e.with_src(()).with_src(source)); + } + }; + + // SAFETY: `remainder` is constructed as a subset of `source`, and so it + // cannot have a larger size than `source`. Both of their `len` methods + // measure bytes (`source` deref's to `[u8]`, and `remainder` is a + // `Ptr<[u8]>`), so `source.len() >= remainder.len()`. Thus, this cannot + // underflow. + #[allow(unstable_name_collisions)] + let split_at = unsafe { source.len().unchecked_sub(remainder.len()) }; + let (bytes, suffix) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + // SAFETY: `try_cast_into` validates size and alignment, and returns a + // `split_at` that indicates how many bytes of `source` correspond to a + // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we + // can rely on `split_at` to produce the correct `source` and `suffix`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((r, suffix)) + } + + /// Constructs a `Ref` from the suffix of a byte slice. + /// + /// This method computes the [largest possible size of `T`][valid-size] that + /// can fit in the trailing bytes of `source`, then attempts to return both + /// a `Ref` to those bytes, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if that suffix of `source` is not + /// appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_suffix(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_suffix(source: B) -> Result<(B, Ref), CastError> { + static_assert_dst_is_not_zst!(T); + let remainder = match Ptr::from_ref(source.deref()) + .try_cast_into::(CastType::Suffix, None) + { + Ok((_, remainder)) => remainder, + Err(e) => { + let e = e.with_src(()); + return Err(e.with_src(source)); + } + }; + + let split_at = remainder.len(); + let (prefix, bytes) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + // SAFETY: `try_cast_into` validates size and alignment, and returns a + // `split_at` that indicates how many bytes of `source` correspond to a + // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we + // can rely on `split_at` to produce the correct `prefix` and `bytes`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((prefix, r)) + } +} + +impl Ref +where + B: ByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the given bytes with DST length equal to `count` + /// without copying. + /// + /// This method attempts to return a `Ref` to the prefix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the remaining bytes. If the length of `source` is not equal to the size + /// of `Self` with `count` elements, or if `source` is not appropriately + /// aligned, this returns `Err`. If [`T: Unaligned`][t-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_bytes_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_bytes_with_elems(source: B, count: usize) -> Result, CastError> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + if source.len() != expected_len { + return Err(SizeError::new(source).into()); + } + Self::from_bytes(source) + } +} + +impl Ref +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the prefix of the given bytes with DST + /// length equal to `count` without copying. + /// + /// This method attempts to return a `Ref` to the prefix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the remaining bytes. If there are insufficient bytes, or if `source` is + /// not appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_prefix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_prefix_with_elems( + source: B, + count: usize, + ) -> Result<(Ref, B), CastError> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + let (prefix, bytes) = source.split_at(expected_len).map_err(SizeError::new)?; + Self::from_bytes(prefix).map(move |l| (l, bytes)) + } + + /// Constructs a `Ref` from the suffix of the given bytes with DST length + /// equal to `count` without copying. + /// + /// This method attempts to return a `Ref` to the suffix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the preceding bytes. If there are insufficient bytes, or if that suffix + /// of `source` is not appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_suffix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_suffix_with_elems( + source: B, + count: usize, + ) -> Result<(B, Ref), CastError> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + let split_at = if let Some(split_at) = source.len().checked_sub(expected_len) { + split_at + } else { + return Err(SizeError::new(source).into()); + }; + // SAFETY: The preceding `source.len().checked_sub(expected_len)` + // guarantees that `split_at` is in-bounds. + let (bytes, suffix) = unsafe { source.split_at_unchecked(split_at) }; + Self::from_bytes(suffix).map(move |l| (bytes, l)) + } +} + +impl<'a, B, T> Ref +where + B: 'a + IntoByteSlice<'a>, + T: FromBytes + KnownLayout + Immutable + ?Sized, +{ + /// Converts this `Ref` into a reference. + /// + /// `into_ref` consumes the `Ref`, and returns a reference to `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::into_ref(r)` instead of `r.into_ref()`. This is so that + /// there is no conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline(always)] + pub fn into_ref(r: Self) -> &'a T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `IntoByteSlice`. + let b = unsafe { r.into_byte_slice() }; + let b = b.into_byte_slice(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_ref(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::()`. + let ptr = unsafe { cast_for_sized::(ptr) }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::() }; + return ptr.as_ref(); + } + + // PANICS: By post-condition on `into_byte_slice`, `b`'s size and + // alignment are valid for `T`. By post-condition, `b.into_byte_slice()` + // produces a byte slice with identical address and length to that + // produced by `b.deref()`. + let ptr = Ptr::from_ref(b.into_byte_slice()) + .try_cast_into_no_leftover::(None) + .expect("zerocopy internal error: into_ref should be infallible"); + let ptr = ptr.recall_validity(); + ptr.as_ref() + } +} + +impl<'a, B, T> Ref +where + B: 'a + IntoByteSliceMut<'a>, + T: FromBytes + IntoBytes + KnownLayout + ?Sized, +{ + /// Converts this `Ref` into a mutable reference. + /// + /// `into_mut` consumes the `Ref`, and returns a mutable reference to `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::into_mut(r)` instead of `r.into_mut()`. This is so that + /// there is no conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline(always)] + pub fn into_mut(r: Self) -> &'a mut T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `IntoByteSliceMut`. + let b = unsafe { r.into_byte_slice_mut() }; + let b = b.into_byte_slice_mut(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_mut(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::()`. + let ptr = unsafe { + cast_for_sized::< + T, + _, + (BecauseRead, BecauseExclusive), + (BecauseMutationCompatible, BecauseInvariantsEq), + >(ptr) + }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::() }; + return ptr.as_mut(); + } + + // PANICS: By post-condition on `into_byte_slice_mut`, `b`'s size and + // alignment are valid for `T`. By post-condition, + // `b.into_byte_slice_mut()` produces a byte slice with identical + // address and length to that produced by `b.deref_mut()`. + let ptr = Ptr::from_mut(b.into_byte_slice_mut()) + .try_cast_into_no_leftover::(None) + .expect("zerocopy internal error: into_ref should be infallible"); + let ptr = ptr.recall_validity::<_, (_, (_, _))>(); + ptr.as_mut() + } +} + +impl Ref +where + B: ByteSlice, + T: ?Sized, +{ + /// Gets the underlying bytes. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::bytes(r)` instead of `r.bytes()`. This is so that there is + /// no conflict with a method on the inner type. + #[inline] + pub fn bytes(r: &Self) -> &[u8] { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + unsafe { r.as_byte_slice().deref() } + } +} + +impl Ref +where + B: ByteSliceMut, + T: ?Sized, +{ + /// Gets the underlying bytes mutably. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::bytes_mut(r)` instead of `r.bytes_mut()`. This is so that + /// there is no conflict with a method on the inner type. + #[inline] + pub fn bytes_mut(r: &mut Self) -> &mut [u8] { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + unsafe { r.as_byte_slice_mut().deref_mut() } + } +} + +impl Ref +where + B: ByteSlice, + T: FromBytes, +{ + /// Reads a copy of `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::read(r)` instead of `r.read()`. This is so that there is no + /// conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline] + pub fn read(r: &Self) -> T { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + let b = unsafe { r.as_byte_slice() }; + + // SAFETY: By postcondition on `as_byte_slice`, we know that `b` is a + // valid size and alignment for `T`. By safety invariant on `ByteSlice`, + // we know that this is preserved via `.deref()`. Because `T: + // FromBytes`, it is sound to interpret these bytes as a `T`. + unsafe { ptr::read(b.deref().as_ptr().cast::()) } + } +} + +impl Ref +where + B: ByteSliceMut, + T: IntoBytes, +{ + /// Writes the bytes of `t` and then forgets `t`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::write(r, t)` instead of `r.write(t)`. This is so that there + /// is no conflict with a method on the inner type. + #[inline] + pub fn write(r: &mut Self, t: T) { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + let b = unsafe { r.as_byte_slice_mut() }; + + // SAFETY: By postcondition on `as_byte_slice_mut`, we know that `b` is + // a valid size and alignment for `T`. By safety invariant on + // `ByteSlice`, we know that this is preserved via `.deref()`. Writing + // `t` to the buffer will allow all of the bytes of `t` to be accessed + // as a `[u8]`, but because `T: IntoBytes`, we know that this is sound. + unsafe { ptr::write(b.deref_mut().as_mut_ptr().cast::(), t) } + } +} + +impl Deref for Ref +where + B: ByteSlice, + T: FromBytes + KnownLayout + Immutable + ?Sized, +{ + type Target = T; + #[inline] + fn deref(&self) -> &T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + let b = unsafe { self.as_byte_slice() }; + let b = b.deref(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_ref(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::()`. + let ptr = unsafe { cast_for_sized::(ptr) }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::() }; + return ptr.as_ref(); + } + + // PANICS: By postcondition on `as_byte_slice`, `b`'s size and alignment + // are valid for `T`, and by invariant on `ByteSlice`, these are + // preserved through `.deref()`, so this `unwrap` will not panic. + let ptr = Ptr::from_ref(b) + .try_cast_into_no_leftover::(None) + .expect("zerocopy internal error: Deref::deref should be infallible"); + let ptr = ptr.recall_validity(); + ptr.as_ref() + } +} + +impl DerefMut for Ref +where + B: ByteSliceMut, + // FIXME(#251): We can't remove `Immutable` here because it's required by + // the impl of `Deref`, which is a super-trait of `DerefMut`. Maybe we can + // add a separate inherent method for this? + T: FromBytes + IntoBytes + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn deref_mut(&mut self) -> &mut T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + let b = unsafe { self.as_byte_slice_mut() }; + let b = b.deref_mut(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_mut(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::()`. + let ptr = unsafe { + cast_for_sized::< + T, + _, + (BecauseRead, BecauseExclusive), + (BecauseMutationCompatible, BecauseInvariantsEq), + >(ptr) + }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::() }; + return ptr.as_mut(); + } + + // PANICS: By postcondition on `as_byte_slice_mut`, `b`'s size and + // alignment are valid for `T`, and by invariant on `ByteSlice`, these + // are preserved through `.deref_mut()`, so this `unwrap` will not + // panic. + let ptr = Ptr::from_mut(b) + .try_cast_into_no_leftover::(None) + .expect("zerocopy internal error: DerefMut::deref_mut should be infallible"); + let ptr = ptr.recall_validity::<_, (_, (_, BecauseExclusive))>(); + ptr.as_mut() + } +} + +impl Display for Ref +where + B: ByteSlice, + T: FromBytes + Display + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + let inner: &T = self; + inner.fmt(fmt) + } +} + +impl Debug for Ref +where + B: ByteSlice, + T: FromBytes + Debug + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + let inner: &T = self; + fmt.debug_tuple("Ref").field(&inner).finish() + } +} + +impl Eq for Ref +where + B: ByteSlice, + T: FromBytes + Eq + KnownLayout + Immutable + ?Sized, +{ +} + +impl PartialEq for Ref +where + B: ByteSlice, + T: FromBytes + PartialEq + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl Ord for Ref +where + B: ByteSlice, + T: FromBytes + Ord + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + let inner: &T = self; + let other_inner: &T = other; + inner.cmp(other_inner) + } +} + +impl PartialOrd for Ref +where + B: ByteSlice, + T: FromBytes + PartialOrd + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + let inner: &T = self; + let other_inner: &T = other; + inner.partial_cmp(other_inner) + } +} + +/// # Safety +/// +/// `T: Sized` and `ptr`'s referent must have size `size_of::()`. +#[inline(always)] +unsafe fn cast_for_sized<'a, T, A, R, S>( + ptr: Ptr<'a, [u8], (A, Aligned, Valid)>, +) -> Ptr<'a, T, (A, Unaligned, Valid)> +where + T: FromBytes + KnownLayout + ?Sized, + A: crate::invariant::Aliasing, + [u8]: MutationCompatible, + T: TransmuteFromPtr, +{ + use crate::pointer::cast::{Cast, Project}; + + enum CastForSized {} + + // SAFETY: `CastForSized` is only used below with the input `ptr`, which the + // caller promises has size `size_of::()`. Thus, the referent produced in + // this cast has the same size as `ptr`'s referent. All operations preserve + // provenance. + unsafe impl Project<[u8], T> for CastForSized { + #[inline(always)] + fn project(src: PtrInner<'_, [u8]>) -> *mut T { + T::raw_from_ptr_len( + src.as_non_null().cast(), + ::from_elem_count(0), + ) + .as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl Cast<[u8], T> for CastForSized {} + + ptr.recall_validity::() + .cast::<_, CastForSized, _>() + .recall_validity::() +} + +#[cfg(test)] +#[allow(clippy::assertions_on_result_states)] +mod tests { + use core::convert::TryInto as _; + + use super::*; + use crate::util::testutil::*; + + #[test] + fn test_mut_slice_into_ref() { + // Prior to #1260/#1299, calling `into_ref` on a `&mut [u8]`-backed + // `Ref` was not supported. + let mut buf = [0u8]; + let r = Ref::<&mut [u8], u8>::from_bytes(&mut buf).unwrap(); + assert_eq!(Ref::into_ref(r), &0); + } + + #[test] + fn test_address() { + // Test that the `Deref` and `DerefMut` implementations return a + // reference which points to the right region of memory. + + let buf = [0]; + let r = Ref::<_, u8>::from_bytes(&buf[..]).unwrap(); + let buf_ptr = buf.as_ptr(); + let deref_ptr: *const u8 = r.deref(); + assert_eq!(buf_ptr, deref_ptr); + + let buf = [0]; + let r = Ref::<_, [u8]>::from_bytes(&buf[..]).unwrap(); + let buf_ptr = buf.as_ptr(); + let deref_ptr = r.deref().as_ptr(); + assert_eq!(buf_ptr, deref_ptr); + } + + // Verify that values written to a `Ref` are properly shared between the + // typed and untyped representations, that reads via `deref` and `read` + // behave the same, and that writes via `deref_mut` and `write` behave the + // same. + fn test_new_helper(mut r: Ref<&mut [u8], AU64>) { + // assert that the value starts at 0 + assert_eq!(*r, AU64(0)); + assert_eq!(Ref::read(&r), AU64(0)); + + // Assert that values written to the typed value are reflected in the + // byte slice. + const VAL1: AU64 = AU64(0xFF00FF00FF00FF00); + *r = VAL1; + assert_eq!(Ref::bytes(&r), &VAL1.to_bytes()); + *r = AU64(0); + Ref::write(&mut r, VAL1); + assert_eq!(Ref::bytes(&r), &VAL1.to_bytes()); + + // Assert that values written to the byte slice are reflected in the + // typed value. + const VAL2: AU64 = AU64(!VAL1.0); // different from `VAL1` + Ref::bytes_mut(&mut r).copy_from_slice(&VAL2.to_bytes()[..]); + assert_eq!(*r, VAL2); + assert_eq!(Ref::read(&r), VAL2); + } + + // Verify that values written to a `Ref` are properly shared between the + // typed and untyped representations; pass a value with `typed_len` `AU64`s + // backed by an array of `typed_len * 8` bytes. + fn test_new_helper_slice(mut r: Ref<&mut [u8], [AU64]>, typed_len: usize) { + // Assert that the value starts out zeroed. + assert_eq!(&*r, vec![AU64(0); typed_len].as_slice()); + + // Check the backing storage is the exact same slice. + let untyped_len = typed_len * 8; + assert_eq!(Ref::bytes(&r).len(), untyped_len); + assert_eq!(Ref::bytes(&r).as_ptr(), r.as_ptr().cast::()); + + // Assert that values written to the typed value are reflected in the + // byte slice. + const VAL1: AU64 = AU64(0xFF00FF00FF00FF00); + for typed in &mut *r { + *typed = VAL1; + } + assert_eq!(Ref::bytes(&r), VAL1.0.to_ne_bytes().repeat(typed_len).as_slice()); + + // Assert that values written to the byte slice are reflected in the + // typed value. + const VAL2: AU64 = AU64(!VAL1.0); // different from VAL1 + Ref::bytes_mut(&mut r).copy_from_slice(&VAL2.0.to_ne_bytes().repeat(typed_len)); + assert!(r.iter().copied().all(|x| x == VAL2)); + } + + #[test] + fn test_new_aligned_sized() { + // Test that a properly-aligned, properly-sized buffer works for new, + // new_from_prefix, and new_from_suffix, and that new_from_prefix and + // new_from_suffix return empty slices. Test that a properly-aligned + // buffer whose length is a multiple of the element size works for + // new_slice. + + // A buffer with an alignment of 8. + let mut buf = Align::<[u8; 8], AU64>::default(); + // `buf.t` should be aligned to 8, so this should always succeed. + test_new_helper(Ref::<_, AU64>::from_bytes(&mut buf.t[..]).unwrap()); + { + // In a block so that `r` and `suffix` don't live too long. + buf.set_default(); + let (r, suffix) = Ref::<_, AU64>::from_prefix(&mut buf.t[..]).unwrap(); + assert!(suffix.is_empty()); + test_new_helper(r); + } + { + buf.set_default(); + let (prefix, r) = Ref::<_, AU64>::from_suffix(&mut buf.t[..]).unwrap(); + assert!(prefix.is_empty()); + test_new_helper(r); + } + + // A buffer with alignment 8 and length 24. We choose this length very + // intentionally: if we instead used length 16, then the prefix and + // suffix lengths would be identical. In the past, we used length 16, + // which resulted in this test failing to discover the bug uncovered in + // #506. + let mut buf = Align::<[u8; 24], AU64>::default(); + // `buf.t` should be aligned to 8 and have a length which is a multiple + // of `size_of::()`, so this should always succeed. + test_new_helper_slice(Ref::<_, [AU64]>::from_bytes(&mut buf.t[..]).unwrap(), 3); + buf.set_default(); + let r = Ref::<_, [AU64]>::from_bytes_with_elems(&mut buf.t[..], 3).unwrap(); + test_new_helper_slice(r, 3); + + let ascending: [u8; 24] = (0..24).collect::>().try_into().unwrap(); + // 16 ascending bytes followed by 8 zeros. + let mut ascending_prefix = ascending; + ascending_prefix[16..].copy_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0]); + // 8 zeros followed by 16 ascending bytes. + let mut ascending_suffix = ascending; + ascending_suffix[..8].copy_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0]); + { + buf.t = ascending_suffix; + let (r, suffix) = Ref::<_, [AU64]>::from_prefix_with_elems(&mut buf.t[..], 1).unwrap(); + assert_eq!(suffix, &ascending[8..]); + test_new_helper_slice(r, 1); + } + { + buf.t = ascending_prefix; + let (prefix, r) = Ref::<_, [AU64]>::from_suffix_with_elems(&mut buf.t[..], 1).unwrap(); + assert_eq!(prefix, &ascending[..16]); + test_new_helper_slice(r, 1); + } + } + + #[test] + fn test_new_oversized() { + // Test that a properly-aligned, overly-sized buffer works for + // `new_from_prefix` and `new_from_suffix`, and that they return the + // remainder and prefix of the slice respectively. + + let mut buf = Align::<[u8; 16], AU64>::default(); + { + // In a block so that `r` and `suffix` don't live too long. `buf.t` + // should be aligned to 8, so this should always succeed. + let (r, suffix) = Ref::<_, AU64>::from_prefix(&mut buf.t[..]).unwrap(); + assert_eq!(suffix.len(), 8); + test_new_helper(r); + } + { + buf.set_default(); + // `buf.t` should be aligned to 8, so this should always succeed. + let (prefix, r) = Ref::<_, AU64>::from_suffix(&mut buf.t[..]).unwrap(); + assert_eq!(prefix.len(), 8); + test_new_helper(r); + } + } + + #[test] + #[allow(clippy::cognitive_complexity)] + fn test_new_error() { + // Fail because the buffer is too large. + + // A buffer with an alignment of 8. + let buf = Align::<[u8; 16], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[..]).is_err()); + + // Fail because the buffer is too small. + + // A buffer with an alignment of 8. + let buf = Align::<[u8; 4], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[..]).is_err()); + assert!(Ref::<_, AU64>::from_prefix(&buf.t[..]).is_err()); + assert!(Ref::<_, AU64>::from_suffix(&buf.t[..]).is_err()); + + // Fail because the length is not a multiple of the element size. + + let buf = Align::<[u8; 12], AU64>::default(); + // `buf.t` has length 12, but element size is 8. + assert!(Ref::<_, [AU64]>::from_bytes(&buf.t[..]).is_err()); + + // Fail because the buffer is too short. + let buf = Align::<[u8; 12], AU64>::default(); + // `buf.t` has length 12, but the element size is 8 (and we're expecting + // two of them). For each function, we test with a length that would + // cause the size to overflow `usize`, and with a normal length that + // will fail thanks to the buffer being too short; these are different + // error paths, and while the error types are the same, the distinction + // shows up in code coverage metrics. + let n = (usize::MAX / mem::size_of::()) + 1; + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[..], 2).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], 2).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], 2).is_err()); + + // Fail because the alignment is insufficient. + + // A buffer with an alignment of 8. An odd buffer size is chosen so that + // the last byte of the buffer has odd alignment. + let buf = Align::<[u8; 13], AU64>::default(); + // Slicing from 1, we get a buffer with size 12 (so the length check + // should succeed) but an alignment of only 1, which is insufficient. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[1..]).is_err()); + assert!(Ref::<_, AU64>::from_prefix(&buf.t[1..]).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes(&buf.t[1..]).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[1..], 1).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[1..], 1).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[1..], 1).is_err()); + // Slicing is unnecessary here because `new_from_suffix` uses the suffix + // of the slice, which has odd alignment. + assert!(Ref::<_, AU64>::from_suffix(&buf.t[..]).is_err()); + + // Fail due to arithmetic overflow. + + let buf = Align::<[u8; 16], AU64>::default(); + let unreasonable_len = usize::MAX / mem::size_of::() + 1; + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], unreasonable_len).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], unreasonable_len).is_err()); + } + + #[test] + #[allow(unstable_name_collisions)] + #[allow(clippy::as_conversions)] + fn test_into_ref_mut() { + #[allow(unused)] + use crate::util::AsAddress as _; + + let mut buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, u64>::from_bytes(&buf.t[..]).unwrap(); + let rf = Ref::into_ref(r); + assert_eq!(rf, &0u64); + let buf_addr = (&buf.t as *const [u8; 8]).addr(); + assert_eq!((rf as *const u64).addr(), buf_addr); + + let r = Ref::<_, u64>::from_bytes(&mut buf.t[..]).unwrap(); + let rf = Ref::into_mut(r); + assert_eq!(rf, &mut 0u64); + assert_eq!((rf as *mut u64).addr(), buf_addr); + + *rf = u64::MAX; + assert_eq!(buf.t, [0xFF; 8]); + } + + #[test] + fn test_display_debug() { + let buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, u64>::from_bytes(&buf.t[..]).unwrap(); + assert_eq!(format!("{}", r), "0"); + assert_eq!(format!("{:?}", r), "Ref(0)"); + + let buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, [u64]>::from_bytes(&buf.t[..]).unwrap(); + assert_eq!(format!("{:?}", r), "Ref([0])"); + } + + #[test] + fn test_eq() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 0_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert_eq!(r1, r2); + } + + #[test] + fn test_ne() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 1_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert_ne!(r1, r2); + } + + #[test] + fn test_ord() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 1_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert!(r1 < r2); + assert_eq!(PartialOrd::partial_cmp(&r1, &r2), Some(Ordering::Less)); + assert_eq!(Ord::cmp(&r1, &r2), Ordering::Less); + } +} + +#[cfg(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS))] +mod benches { + use test::{self, Bencher}; + + use super::*; + use crate::util::testutil::*; + + #[bench] + fn bench_from_bytes_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + // `buf.t` should be aligned to 8, so this should always succeed. + let bytes = &buf.t[..]; + b.iter(|| test::black_box(Ref::<_, AU64>::from_bytes(test::black_box(bytes)).unwrap())); + } + + #[bench] + fn bench_into_ref_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + let bytes = &buf.t[..]; + let r = Ref::<_, AU64>::from_bytes(bytes).unwrap(); + b.iter(|| test::black_box(Ref::into_ref(test::black_box(r)))); + } + + #[bench] + fn bench_into_mut_sized(b: &mut Bencher) { + let mut buf = Align::<[u8; 8], AU64>::default(); + let buf = &mut buf.t[..]; + let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); + b.iter(move || { + // SAFETY: The preceding `from_bytes` succeeded, and so we know that + // `buf` is validly-aligned and has the correct length. + let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + test::black_box(Ref::into_mut(test::black_box(r))); + }); + } + + #[bench] + fn bench_deref_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + let bytes = &buf.t[..]; + let r = Ref::<_, AU64>::from_bytes(bytes).unwrap(); + b.iter(|| { + let temp = test::black_box(r); + test::black_box(temp.deref()); + }); + } + + #[bench] + fn bench_deref_mut_sized(b: &mut Bencher) { + let mut buf = Align::<[u8; 8], AU64>::default(); + let buf = &mut buf.t[..]; + let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); + b.iter(|| { + // SAFETY: The preceding `from_bytes` succeeded, and so we know that + // `buf` is validly-aligned and has the correct length. + let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + let mut temp = test::black_box(r); + test::black_box(temp.deref_mut()); + }); + } +} diff --git a/rust/zerocopy/src/split_at.rs b/rust/zerocopy/src/split_at.rs new file mode 100644 index 000000000000..1d7b2c7d6032 --- /dev/null +++ b/rust/zerocopy/src/split_at.rs @@ -0,0 +1,1088 @@ +// Copyright 2025 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use super::*; +use crate::pointer::invariant::{Aligned, Exclusive, Invariants, Shared, Valid}; + +/// Types that can be split in two. +/// +/// This trait generalizes Rust's existing support for splitting slices to +/// support slices and slice-based dynamically-sized types ("slice DSTs"). +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(SplitAt)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{SplitAt, KnownLayout}; +/// #[derive(SplitAt, KnownLayout)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ..., +/// # */ +/// // `SplitAt` types must have at least one field. +/// field: T, +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `SplitAt`. +/// +/// # Safety +/// +/// This trait does not convey any safety guarantees to code outside this crate. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `SplitAt`. Future +/// releases of zerocopy may make backwards-breaking changes to these items, +/// including changes that only affect soundness, which may cause code which +/// uses those items to silently become unsound. +/// +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::SplitAt")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.SplitAt.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(SplitAt)]` to `{Self}`") +)] +// # Safety +// +// The trailing slice is well-aligned for its element type. `Self` is `[T]`, or +// a `repr(C)` or `repr(transparent)` slice DST. +pub unsafe trait SplitAt: KnownLayout { + /// The element type of the trailing slice. + type Elem; + + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Unsafely splits `self` in two. + /// + /// # Safety + /// + /// The caller promises that `l_len` is not greater than the length of + /// `self`'s trailing slice. + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_at_unchecked", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline] + #[must_use] + unsafe fn split_at_unchecked(&self, l_len: usize) -> Split<&Self> { + // SAFETY: By precondition on the caller, `l_len <= self.len()`. + unsafe { Split::<&Self>::new(self, l_len) } + } + + /// Attempts to split `self` in two. + /// + /// Returns `None` if `l_len` is greater than the length of `self`'s + /// trailing slice. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Immutable` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_immutable(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_at", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline] + #[must_use = "has no side effects"] + fn split_at(&self, l_len: usize) -> Option> { + MetadataOf::new_in_bounds(self, l_len).map( + #[inline(always)] + |l_len| { + // SAFETY: We have ensured that `l_len <= self.len()` (by + // post-condition on `MetadataOf::new_in_bounds`) + unsafe { Split::new(self, l_len.get()) } + }, + ) + } + + /// Unsafely splits `self` in two. + /// + /// # Safety + /// + /// The caller promises that `l_len` is not greater than the length of + /// `self`'s trailing slice. + /// + #[doc = codegen_header!("h5", "split_at_mut_unchecked")] + /// + /// See [`SplitAt::split_at_unchecked`](#method.split_at_unchecked.codegen). + #[inline] + #[must_use] + unsafe fn split_at_mut_unchecked(&mut self, l_len: usize) -> Split<&mut Self> { + // SAFETY: By precondition on the caller, `l_len <= self.len()`. + unsafe { Split::<&mut Self>::new(self, l_len) } + } + + /// Attempts to split `self` in two. + /// + /// Returns `None` if `l_len` is greater than the length of `self`'s + /// trailing slice, or if the given `l_len` would result in [the trailing + /// padding](KnownLayout#slice-dst-layout) of the left portion overlapping + /// the right portion. + /// + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_at_mut")] + /// + /// See [`SplitAt::split_at`](#method.split_at.codegen). + #[inline] + fn split_at_mut(&mut self, l_len: usize) -> Option> { + MetadataOf::new_in_bounds(self, l_len).map( + #[inline(always)] + |l_len| { + // SAFETY: We have ensured that `l_len <= self.len()` (by + // post-condition on `MetadataOf::new_in_bounds`) + unsafe { Split::new(self, l_len.get()) } + }, + ) + } +} + +// SAFETY: `[T]`'s trailing slice is `[T]`, which is trivially aligned. +unsafe impl SplitAt for [T] { + type Elem = T; + + #[inline] + #[allow(dead_code)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } +} + +/// A `T` that has been split into two possibly-overlapping parts. +/// +/// For some dynamically sized types, the padding that appears after the +/// trailing slice field [is a dynamic function of the trailing slice +/// length](KnownLayout#slice-dst-layout). If `T` is split at a length that +/// requires trailing padding, the trailing padding of the left part of the +/// split `T` will overlap the right part. If `T` is a mutable reference or +/// permits interior mutation, you must ensure that the left and right parts do +/// not overlap. You can do this at zero-cost using using +/// [`Self::via_immutable`], [`Self::via_into_bytes`], or +/// [`Self::via_unaligned`], or with a dynamic check by using +/// [`Self::via_runtime_check`]. +#[derive(Debug)] +pub struct Split { + /// A pointer to the source slice DST. + source: T, + /// The length of the future left half of `source`. + /// + /// # Safety + /// + /// If `source` is a pointer to a slice DST, `l_len` is no greater than + /// `source`'s length. + l_len: usize, +} + +impl Split { + /// Produces a `Split` of `source` with `l_len`. + /// + /// # Safety + /// + /// `l_len` is no greater than `source`'s length. + #[inline(always)] + unsafe fn new(source: T, l_len: usize) -> Self { + Self { source, l_len } + } +} + +impl<'a, T> Split<&'a T> +where + T: ?Sized + SplitAt, +{ + #[inline(always)] + fn into_ptr(self) -> Split> { + let source = Ptr::from_ref(self.source); + // SAFETY: `Ptr::from_ref(self.source)` points to exactly `self.source` + // and thus maintains the invariants of `self` with respect to `l_len`. + unsafe { Split::new(source, self.l_len) } + } + + /// Produces the split parts of `self`, using [`Immutable`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Immutable` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_immutable(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_immutable", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_immutable(self) -> (&'a T, &'a [T::Elem]) + where + T: Immutable, + { + let (l, r) = self.into_ptr().via_immutable(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, IntoBytes)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_via_into_bytes")] + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_into_bytes(self) -> (&'a T, &'a [T::Elem]) + where + T: IntoBytes, + { + let (l, r) = self.into_ptr().via_into_bytes(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Unaligned` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_unaligned(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_via_unaligned")] + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_unaligned(self) -> (&'a T, &'a [T::Elem]) + where + T: Unaligned, + { + let (l, r) = self.into_ptr().via_unaligned(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_immutable`], [`Self::via_into_bytes`], or + /// [`Self::via_unaligned`], which have no runtime cost. + /// + /// Note that this check is overly conservative if `T` is [`Immutable`]; for + /// some types, this check will reject some splits which + /// [`Self::via_immutable`] will accept. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes, IntoBytes, network_endian::U16}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, Debug)] + /// #[repr(C, align(2))] + /// struct Packet { + /// length: U16, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = [ + /// 4u16.to_be(), + /// 1u16.to_be(), + /// 2u16.to_be(), + /// 3u16.to_be(), + /// 4u16.to_be() + /// ]; + /// + /// let packet = Packet::ref_from_bytes(bytes.as_bytes()).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [0, 1, 0, 2, 0, 3, 0, 4]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length.into()).unwrap(); + /// + /// // Use a dynamic check to prove that it's okay to return concurrent + /// // references to `packet` and `rest`. + /// let (packet, rest) = split.via_runtime_check().unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [0, 1, 0, 2]); + /// assert_eq!(rest, [0, 3, 0, 4]); + /// + /// // Attempt to split `packet` at `length - 1`. + /// let idx = packet.length.get() - 1; + /// let split = packet.split_at(idx as usize).unwrap(); + /// + /// // Attempt (and fail) to use a dynamic check to prove that it's okay + /// // to return concurrent references to `packet` and `rest`. Note that + /// // this is a case of `via_runtime_check` being overly conservative. + /// // Although the left and right parts indeed overlap, the `Immutable` + /// // bound ensures that concurrently referencing these overlapping + /// // parts is sound. + /// assert!(split.via_runtime_check().is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_runtime_check", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_runtime_check(self) -> Result<(&'a T, &'a [T::Elem]), Self> { + match self.into_ptr().via_runtime_check() { + Ok((l, r)) => Ok((l.as_ref(), r.as_ref())), + Err(s) => Err(s.into_ref()), + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// If `T` permits interior mutation, the trailing padding bytes of the left + /// portion must not overlap the right portion. For some dynamically sized + /// types, the padding that appears after the trailing slice field [is a + /// dynamic function of the trailing slice + /// length](KnownLayout#slice-dst-layout). Thus, for some types, this + /// condition is dependent on the length of the left portion. + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_unchecked", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub unsafe fn via_unchecked(self) -> (&'a T, &'a [T::Elem]) { + // SAFETY: The aliasing of `self.into_ptr()` is not `Exclusive`, but the + // caller has promised that if `T` permits interior mutation then the + // left and right portions of `self` split at `l_len` do not overlap. + let (l, r) = unsafe { self.into_ptr().via_unchecked() }; + (l.as_ref(), r.as_ref()) + } +} + +impl<'a, T> Split<&'a mut T> +where + T: ?Sized + SplitAt, +{ + #[inline(always)] + fn into_ptr(self) -> Split> { + let source = Ptr::from_mut(self.source); + // SAFETY: `Ptr::from_mut(self.source)` points to exactly `self.source`, + // and thus maintains the invariants of `self` with respect to `l_len`. + unsafe { Split::new(source, self.l_len) } + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_into_bytes(self) -> (&'a mut T, &'a mut [T::Elem]) + where + T: IntoBytes, + { + let (l, r) = self.into_ptr().via_into_bytes(); + (l.as_mut(), r.as_mut()) + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes, Unaligned)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `Unaligned` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_unaligned(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_unaligned(self) -> (&'a mut T, &'a mut [T::Elem]) + where + T: Unaligned, + { + let (l, r) = self.into_ptr().via_unaligned(); + (l.as_mut(), r.as_mut()) + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_into_bytes`] or [`Self::via_unaligned`], which + /// have no runtime cost. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes, Debug)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use a dynamic check to prove that it's okay to return concurrent + /// // references to `packet` and `rest`. + /// let (packet, rest) = split.via_runtime_check().unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_runtime_check`](#method.split_via_runtime_check.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_runtime_check(self) -> Result<(&'a mut T, &'a mut [T::Elem]), Self> { + match self.into_ptr().via_runtime_check() { + Ok((l, r)) => Ok((l.as_mut(), r.as_mut())), + Err(s) => Err(s.into_mut()), + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// The trailing padding bytes of the left portion must not overlap the + /// right portion. For some dynamically sized types, the padding that + /// appears after the trailing slice field [is a dynamic function of the + /// trailing slice length](KnownLayout#slice-dst-layout). Thus, for some + /// types, this condition is dependent on the length of the left portion. + /// + /// # Code Generation + /// + /// See [`Split::via_unchecked`](#method.split_via_unchecked.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub unsafe fn via_unchecked(self) -> (&'a mut T, &'a mut [T::Elem]) { + // SAFETY: The aliasing of `self.into_ptr()` is `Exclusive`, and the + // caller has promised that the left and right portions of `self` split + // at `l_len` do not overlap. + let (l, r) = unsafe { self.into_ptr().via_unchecked() }; + (l.as_mut(), r.as_mut()) + } +} + +impl<'a, T, I> Split> +where + T: ?Sized + SplitAt, + I: Invariants, +{ + fn into_ref(self) -> Split<&'a T> + where + I: Invariants, + { + // SAFETY: `self.source.as_ref()` points to exactly the same referent as + // `self.source` and thus maintains the invariants of `self` with + // respect to `l_len`. + unsafe { Split::new(self.source.as_ref(), self.l_len) } + } + + fn into_mut(self) -> Split<&'a mut T> + where + I: Invariants, + { + // SAFETY: `self.source.as_mut()` points to exactly the same referent as + // `self.source` and thus maintains the invariants of `self` with + // respect to `l_len`. + unsafe { Split::new(self.source.unify_invariants().as_mut(), self.l_len) } + } + + /// Produces the length of `self`'s left part. + #[inline(always)] + fn l_len(&self) -> MetadataOf { + // SAFETY: By invariant on `Split`, `self.l_len` is not greater than the + // length of `self.source`. + unsafe { MetadataOf::::new_unchecked(self.l_len) } + } + + /// Produces the split parts of `self`, using [`Immutable`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_immutable(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: Immutable, + I: Invariants, + { + // SAFETY: `Aliasing = Shared` and `T: Immutable`. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_into_bytes(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: IntoBytes, + { + // SAFETY: By `T: IntoBytes`, `T` has no padding for any length. + // Consequently, `T` can be split into non-overlapping parts at any + // index. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_unaligned(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: Unaligned, + { + // SAFETY: By `T: SplitAt + Unaligned`, `T` is either a slice or a + // `repr(C)` or `repr(transparent)` slice DST that is well-aligned at + // any address and length. If `T` is a slice DST with alignment 1, + // `repr(C)` or `repr(transparent)` ensures that no padding is placed + // after the final element of the trailing slice. Consequently, `T` can + // be split into strictly non-overlapping parts any any index. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_immutable`], [`Self::via_into_bytes`], or + /// [`Self::via_unaligned`], which have no runtime cost. + #[inline(always)] + fn via_runtime_check(self) -> Result<(Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>), Self> { + let l_len = self.l_len(); + // FIXME(#1290): Once we require `KnownLayout` on all fields, add an + // `IS_IMMUTABLE` associated const, and add `T::IS_IMMUTABLE ||` to the + // below check. + if l_len.padding_needed_for() == 0 { + // SAFETY: By `T: SplitAt`, `T` is either `[T]`, or a `repr(C)` or + // `repr(transparent)` slice DST, for which the trailing padding + // needed to accommodate `l_len` trailing elements is + // `l_len.padding_needed_for()`. If no trailing padding is required, + // the left and right parts are strictly non-overlapping. + Ok(unsafe { self.via_unchecked() }) + } else { + Err(self) + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// The caller promises that if `I::Aliasing` is [`Exclusive`] or `T` + /// permits interior mutation, then `l_len.padding_needed_for() == 0`. + #[inline(always)] + unsafe fn via_unchecked(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) { + let l_len = self.l_len(); + let inner = self.source.as_inner(); + + // SAFETY: By invariant on `Self::l_len`, `l_len` is not greater than + // the length of `inner`'s trailing slice. + let (left, right) = unsafe { inner.split_at_unchecked(l_len) }; + + // Lemma 0: `left` and `right` conform to the aliasing invariant + // `I::Aliasing`. Proof: If `I::Aliasing` is `Exclusive` or `T` permits + // interior mutation, the caller promises that `l_len.padding_needed_for() + // == 0`. Consequently, by post-condition on `PtrInner::split_at_unchecked`, + // there is no trailing padding after `left`'s final element that would + // overlap into `right`. If `I::Aliasing` is shared and `T` forbids interior + // mutation, then overlap between their referents is permissible. + + // SAFETY: + // 0. `left` conforms to the aliasing invariant of `I::Aliasing`, by Lemma 0. + // 1. `left` conforms to the alignment invariant of `I::Alignment, because + // the referents of `left` and `Self` have the same address and type + // (and, thus, alignment requirement). + // 2. `left` conforms to the validity invariant of `I::Validity`, neither + // the type nor bytes of `left`'s referent have been changed. + let left = unsafe { Ptr::from_inner(left) }; + + // SAFETY: + // 0. `right` conforms to the aliasing invariant of `I::Aliasing`, by Lemma + // 0. + // 1. `right` conforms to the alignment invariant of `I::Alignment, because + // if `ptr` with `I::Alignment = Aligned`, then by invariant on `T: + // SplitAt`, the trailing slice of `ptr` (from which `right` is derived) + // will also be well-aligned. + // 2. `right` conforms to the validity invariant of `I::Validity`, + // because `right: [T::Elem]` is derived from the trailing slice of + // `ptr`, which, by contract on `T: SplitAt::Elem`, has type + // `[T::Elem]`. The `left` part cannot be used to invalidate `right`, + // because the caller promises that if `I::Aliasing` is `Exclusive` + // or `T` permits interior mutation, then `l_len.padding_needed_for() + // == 0` and thus the parts will be non-overlapping. + let right = unsafe { Ptr::from_inner(right) }; + + (left, right) + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "derive")] + #[test] + fn test_split_at() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Debug)] + #[repr(C)] + struct SliceDst { + prefix: [u8; OFFSET], + trailing: [u8], + } + + #[allow(clippy::as_conversions)] + fn test_split_at() { + // Test `split_at` + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::::ref_from_bytes(&arr[..]).unwrap(); + for i in 0..=n { + let (l, r) = dst.split_at(i).unwrap().via_runtime_check().unwrap(); + let l_sum: u8 = l.trailing.iter().sum(); + let r_sum: u8 = r.iter().sum(); + assert_eq!(l_sum, i as u8); + assert_eq!(r_sum, (n - i) as u8); + assert_eq!(l_sum + r_sum, n as u8); + } + + // Test `split_at_mut` + let n: usize = BUFFER_SIZE - OFFSET; + let mut arr = [1; BUFFER_SIZE]; + let dst = SliceDst::::mut_from_bytes(&mut arr[..]).unwrap(); + for i in 0..=n { + let (l, r) = dst.split_at_mut(i).unwrap().via_runtime_check().unwrap(); + let l_sum: u8 = l.trailing.iter().sum(); + let r_sum: u8 = r.iter().sum(); + assert_eq!(l_sum, i as u8); + assert_eq!(r_sum, (n - i) as u8); + assert_eq!(l_sum + r_sum, n as u8); + } + } + + test_split_at::<0, 16>(); + test_split_at::<1, 17>(); + test_split_at::<2, 18>(); + } + + #[cfg(feature = "derive")] + #[test] + #[allow(clippy::as_conversions)] + fn test_split_at_overlapping() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C, align(2))] + struct SliceDst { + prefix: u8, + trailing: [u8], + } + + const N: usize = 16; + + let arr = [1u16; N]; + let dst = SliceDst::ref_from_bytes(arr.as_bytes()).unwrap(); + + for i in 0..N { + let split = dst.split_at(i).unwrap().via_runtime_check(); + if i % 2 == 1 { + assert!(split.is_ok()); + } else { + assert!(split.is_err()); + } + } + } + #[test] + fn test_split_at_unchecked() { + use crate::SplitAt; + let mut arr = [1, 2, 3, 4]; + let slice = &arr[..]; + // SAFETY: 2 <= arr.len() (4) + let split = unsafe { SplitAt::split_at_unchecked(slice, 2) }; + // SAFETY: SplitAt::split_at_unchecked guarantees that the split is valid. + let (l, r) = unsafe { split.via_unchecked() }; + assert_eq!(l, &[1, 2]); + assert_eq!(r, &[3, 4]); + + let slice_mut = &mut arr[..]; + // SAFETY: 2 <= arr.len() (4) + let split = unsafe { SplitAt::split_at_mut_unchecked(slice_mut, 2) }; + // SAFETY: SplitAt::split_at_mut_unchecked guarantees that the split is valid. + let (l, r) = unsafe { split.via_unchecked() }; + assert_eq!(l, &mut [1, 2]); + assert_eq!(r, &mut [3, 4]); + } + + #[test] + fn test_split_at_via_methods() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Debug)] + #[repr(C)] + struct Packet { + length: u8, + body: [u8], + } + + let arr = [1, 2, 3, 4]; + let packet = Packet::ref_from_bytes(&arr[..]).unwrap(); + + let split1 = packet.split_at(2).unwrap(); + let (l, r) = split1.via_immutable(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + + let split2 = packet.split_at(2).unwrap(); + let (l, r) = split2.via_into_bytes(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + } + #[test] + fn test_split_at_via_unaligned() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt, Unaligned}; + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Unaligned)] + #[repr(C)] + struct Packet { + length: u8, + body: [u8], + } + + let arr = [1, 2, 3, 4]; + let packet = Packet::ref_from_bytes(&arr[..]).unwrap(); + + let split = packet.split_at(2).unwrap(); + let (l, r) = split.via_unaligned(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + } +} diff --git a/rust/zerocopy/src/util/macro_util.rs b/rust/zerocopy/src/util/macro_util.rs new file mode 100644 index 000000000000..2235d0b8b49a --- /dev/null +++ b/rust/zerocopy/src/util/macro_util.rs @@ -0,0 +1,1308 @@ +// Copyright 2022 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Utilities used by macros and by `zerocopy-derive`. +//! +//! These are defined here `zerocopy` rather than in code generated by macros or +//! by `zerocopy-derive` so that they can be compiled once rather than +//! recompiled for every invocation (e.g., if they were defined in generated +//! code, then deriving `IntoBytes` and `FromBytes` on three different types +//! would result in the code in question being emitted and compiled six +//! different times). + +#![allow(missing_debug_implementations)] + +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +use core::ptr::{self, NonNull}; +use core::{marker::PhantomData, mem, num::Wrapping}; + +use crate::{ + pointer::{ + cast::CastSized, + invariant::{Aligned, Initialized, Valid}, + BecauseImmutable, + }, + FromBytes, Immutable, IntoBytes, KnownLayout, Ptr, ReadOnly, TryFromBytes, ValidityError, +}; + +/// Projects the type of the field at `Index` in `Self` without regard for field +/// privacy. +/// +/// The `Index` parameter is any sort of handle that identifies the field; its +/// definition is the obligation of the implementer. +/// +/// # Safety +/// +/// Unsafe code may assume that this accurately reflects the definition of +/// `Self`. +pub unsafe trait Field { + /// The type of the field at `Index`. + type Type: ?Sized; +} + +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented( + message = "`{T}` has {PADDING_BYTES} total byte(s) of padding", + label = "types with padding cannot implement `IntoBytes`", + note = "consider using `zerocopy::Unalign` to lower the alignment of individual fields", + note = "consider adding explicit fields where padding would be", + note = "consider using `#[repr(packed)]` to remove padding" + ) +)] +pub trait PaddingFree {} +impl PaddingFree for () {} + +// FIXME(#1112): In the slice DST case, we should delegate to *both* +// `PaddingFree` *and* `DynamicPaddingFree` (and probably rename `PaddingFree` +// to `StaticPaddingFree` or something - or introduce a third trait with that +// name) so that we can have more clear error messages. + +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented( + message = "`{T}` has one or more padding bytes", + label = "types with padding cannot implement `IntoBytes`", + note = "consider using `zerocopy::Unalign` to lower the alignment of individual fields", + note = "consider adding explicit fields where padding would be", + note = "consider using `#[repr(packed)]` to remove padding" + ) +)] +pub trait DynamicPaddingFree {} +impl DynamicPaddingFree for () {} + +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +const _64K: usize = 1 << 16; + +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +#[repr(C, align(65536))] +struct Aligned64kAllocation([u8; _64K]); + +/// A pointer to an aligned allocation of size 2^16. +/// +/// # Safety +/// +/// `ALIGNED_64K_ALLOCATION` is guaranteed to point to the entirety of an +/// allocation with size and alignment 2^16, and to have valid provenance. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +pub const ALIGNED_64K_ALLOCATION: NonNull<[u8]> = { + const REF: &Aligned64kAllocation = &Aligned64kAllocation([0; _64K]); + let ptr: *const Aligned64kAllocation = REF; + let ptr: *const [u8] = ptr::slice_from_raw_parts(ptr.cast(), _64K); + // SAFETY: + // - `ptr` is derived from a Rust reference, which is guaranteed to be + // non-null. + // - `ptr` is derived from an `&Aligned64kAllocation`, which has size and + // alignment `_64K` as promised. Its length is initialized to `_64K`, + // which means that it refers to the entire allocation. + // - `ptr` is derived from a Rust reference, which is guaranteed to have + // valid provenance. + // + // FIXME(#429): Once `NonNull::new_unchecked` docs document that it + // preserves provenance, cite those docs. + // FIXME: Replace this `as` with `ptr.cast_mut()` once our MSRV >= 1.65 + #[allow(clippy::as_conversions)] + unsafe { + NonNull::new_unchecked(ptr as *mut _) + } +}; + +/// Computes the offset of the base of the field `$trailing_field_name` within +/// the type `$ty`. +/// +/// `trailing_field_offset!` produces code which is valid in a `const` context. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! trailing_field_offset { + ($ty:ty, $trailing_field_name:tt) => {{ + let min_size = { + let zero_elems: *const [()] = + $crate::util::macro_util::core_reexport::ptr::slice_from_raw_parts( + $crate::util::macro_util::core_reexport::ptr::NonNull::<()>::dangling() + .as_ptr() + .cast_const(), + 0, + ); + // SAFETY: + // - If `$ty` is `Sized`, `size_of_val_raw` is always safe to call. + // - Otherwise: + // - If `$ty` is not a slice DST, this pointer conversion will + // fail due to "mismatched vtable kinds", and compilation will + // fail. + // - If `$ty` is a slice DST, we have constructed `zero_elems` to + // have zero trailing slice elements. Per the `size_of_val_raw` + // docs, "For the special case where the dynamic tail length is + // 0, this function is safe to call." [1] + // + // [1] https://doc.rust-lang.org/nightly/std/mem/fn.size_of_val_raw.html + unsafe { + #[allow(clippy::as_conversions)] + $crate::util::macro_util::core_reexport::mem::size_of_val_raw( + zero_elems as *const $ty, + ) + } + }; + + assert!(min_size <= _64K); + + #[allow(clippy::as_conversions)] + let ptr = ALIGNED_64K_ALLOCATION.as_ptr() as *const $ty; + + // SAFETY: + // - Thanks to the preceding `assert!`, we know that the value with zero + // elements fits in `_64K` bytes, and thus in the allocation addressed + // by `ALIGNED_64K_ALLOCATION`. The offset of the trailing field is + // guaranteed to be no larger than this size, so this field projection + // is guaranteed to remain in-bounds of its allocation. + // - Because the minimum size is no larger than `_64K` bytes, and + // because an object's size must always be a multiple of its alignment + // [1], we know that `$ty`'s alignment is no larger than `_64K`. The + // allocation addressed by `ALIGNED_64K_ALLOCATION` is guaranteed to + // be aligned to `_64K`, so `ptr` is guaranteed to satisfy `$ty`'s + // alignment. + // - As required by `addr_of!`, we do not write through `field`. + // + // Note that, as of [2], this requirement is technically unnecessary + // for Rust versions >= 1.75.0, but no harm in guaranteeing it anyway + // until we bump our MSRV. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html: + // + // The size of a value is always a multiple of its alignment. + // + // [2] https://github.com/rust-lang/reference/pull/1387 + let field = unsafe { + $crate::util::macro_util::core_reexport::ptr::addr_of!((*ptr).$trailing_field_name) + }; + // SAFETY: + // - Both `ptr` and `field` are derived from the same allocated object. + // - By the preceding safety comment, `field` is in bounds of that + // allocated object. + // - The distance, in bytes, between `ptr` and `field` is required to be + // a multiple of the size of `u8`, which is trivially true because + // `u8`'s size is 1. + // - The distance, in bytes, cannot overflow `isize`. This is guaranteed + // because no allocated object can have a size larger than can fit in + // `isize`. [1] + // - The distance being in-bounds cannot rely on wrapping around the + // address space. This is guaranteed because the same is guaranteed of + // allocated objects. [1] + // + // [1] FIXME(#429), FIXME(https://github.com/rust-lang/rust/pull/116675): + // Once these are guaranteed in the Reference, cite it. + let offset = unsafe { field.cast::().offset_from(ptr.cast::()) }; + // Guaranteed not to be lossy: `field` comes after `ptr`, so the offset + // from `ptr` to `field` is guaranteed to be positive. + assert!(offset >= 0); + Some( + #[allow(clippy::as_conversions)] + { + offset as usize + }, + ) + }}; +} + +/// Computes alignment of `$ty: ?Sized`. +/// +/// `align_of!` produces code which is valid in a `const` context. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! align_of { + ($ty:ty) => {{ + // SAFETY: `OffsetOfTrailingIsAlignment` is `repr(C)`, and its layout is + // guaranteed [1] to begin with the single-byte layout for `_byte`, + // followed by the padding needed to align `_trailing`, then the layout + // for `_trailing`, and finally any trailing padding bytes needed to + // correctly-align the entire struct. + // + // This macro computes the alignment of `$ty` by counting the number of + // bytes preceding `_trailing`. For instance, if the alignment of `$ty` + // is `1`, then no padding is required align `_trailing` and it will be + // located immediately after `_byte` at offset 1. If the alignment of + // `$ty` is 2, then a single padding byte is required before + // `_trailing`, and `_trailing` will be located at offset 2. + + // This correspondence between offset and alignment holds for all valid + // Rust alignments, and we confirm this exhaustively (or, at least up to + // the maximum alignment supported by `trailing_field_offset!`) in + // `test_align_of_dst`. + // + // [1]: https://doc.rust-lang.org/nomicon/other-reprs.html#reprc + + #[repr(C)] + struct OffsetOfTrailingIsAlignment { + _byte: u8, + _trailing: $ty, + } + + trailing_field_offset!(OffsetOfTrailingIsAlignment, _trailing) + }}; +} + +mod size_to_tag { + pub trait SizeToTag { + type Tag; + } + + impl SizeToTag<1> for () { + type Tag = u8; + } + impl SizeToTag<2> for () { + type Tag = u16; + } + impl SizeToTag<4> for () { + type Tag = u32; + } + impl SizeToTag<8> for () { + type Tag = u64; + } + impl SizeToTag<16> for () { + type Tag = u128; + } +} + +/// An alias for the unsigned integer of the given size in bytes. +#[doc(hidden)] +pub type SizeToTag = <() as size_to_tag::SizeToTag>::Tag; + +// We put `Sized` in its own module so it can have the same name as the standard +// library `Sized` without shadowing it in the parent module. +#[cfg(not(no_zerocopy_diagnostic_on_unimplemented_1_78_0))] +mod __size_of { + #[diagnostic::on_unimplemented( + message = "`{Self}` is unsized", + label = "`IntoBytes` needs all field types to be `Sized` in order to determine whether there is padding", + note = "consider using `#[repr(packed)]` to remove padding", + note = "`IntoBytes` does not require the fields of `#[repr(packed)]` types to be `Sized`" + )] + pub trait Sized: core::marker::Sized {} + impl Sized for T {} + + #[inline(always)] + #[must_use] + #[allow(clippy::needless_maybe_sized)] + pub const fn size_of() -> usize { + core::mem::size_of::() + } +} + +#[cfg(no_zerocopy_diagnostic_on_unimplemented_1_78_0)] +pub use core::mem::size_of; + +#[cfg(not(no_zerocopy_diagnostic_on_unimplemented_1_78_0))] +pub use __size_of::size_of; + +/// How many padding bytes does the struct type `$t` have? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a struct +/// type, or else `struct_padding!`'s result may be meaningless. +/// +/// Note that `struct_padding!`'s results are independent of `repcr` since they +/// only consider the size of the type and the sizes of the fields. Whatever the +/// repr, the size of the type already takes into account any padding that the +/// compiler has decided to add. Structs with well-defined representations (such +/// as `repr(C)`) can use this macro to check for padding. Note that while this +/// may yield some consistent value for some `repr(Rust)` structs, it is not +/// guaranteed across platforms or compilations. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! struct_padding { + ($t:ty, $_align:expr, $_packed:expr, [$($ts:ty),*]) => {{ + // The `align` and `packed` directives can be ignored here. Regardless + // of if and how they are set, comparing the size of `$t` to the sum of + // its field sizes is a reliable indicator of the presence of padding. + $crate::util::macro_util::size_of::<$t>() - (0 $(+ $crate::util::macro_util::size_of::<$ts>())*) + }}; +} + +/// Does the `repr(C)` struct type `$t` have padding? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a +/// `repr(C)` struct type, or else `struct_has_padding!`'s result may be +/// meaningless. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! repr_c_struct_has_padding { + ($t:ty, $align:expr, $packed:expr, [$($ts:tt),*]) => {{ + let layout = $crate::DstLayout::for_repr_c_struct( + $align, + $packed, + &[$($crate::repr_c_struct_has_padding!(@field $ts),)*] + ); + layout.requires_static_padding() || layout.requires_dynamic_padding() + }}; + (@field ([$t:ty])) => { + <[$t] as $crate::KnownLayout>::LAYOUT + }; + (@field ($t:ty)) => { + $crate::DstLayout::for_unpadded_type::<$t>() + }; + (@field [$t:ty]) => { + <[$t] as $crate::KnownLayout>::LAYOUT + }; + (@field $t:ty) => { + $crate::DstLayout::for_unpadded_type::<$t>() + }; +} + +/// Does the union type `$t` have padding? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a union +/// type, or else `union_padding!`'s result may be meaningless. +/// +/// Note that `union_padding!`'s results are independent of `repr` since they +/// only consider the size of the type and the sizes of the fields. Whatever the +/// repr, the size of the type already takes into account any padding that the +/// compiler has decided to add. Unions with well-defined representations (such +/// as `repr(C)`) can use this macro to check for padding. Note that while this +/// may yield some consistent value for some `repr(Rust)` unions, it is not +/// guaranteed across platforms or compilations. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! union_padding { + ($t:ty, $_align:expr, $_packed:expr, [$($ts:ty),*]) => {{ + // The `align` and `packed` directives can be ignored here. Regardless + // of if and how they are set, comparing the size of `$t` to each of its + // field sizes is a reliable indicator of the presence of padding. + let mut max = 0; + $({ + let padding = $crate::util::macro_util::size_of::<$t>() - $crate::util::macro_util::size_of::<$ts>(); + if padding > max { + max = padding; + } + })* + max + }}; +} + +/// How many padding bytes does the enum type `$t` have? +/// +/// `$disc` is the type of the enum tag, and `$ts` is a list of fields in each +/// square-bracket-delimited variant. `$t` must be an enum, or else +/// `enum_padding!`'s result may be meaningless. An enum has padding if any of +/// its variant structs [1][2] contain padding, and so all of the variants of an +/// enum must be "full" in order for the enum to not have padding. +/// +/// The results of `enum_padding!` require that the enum is not `repr(Rust)`, as +/// `repr(Rust)` enums may niche the enum's tag and reduce the total number of +/// bytes required to represent the enum as a result. As long as the enum is +/// `repr(C)`, `repr(int)`, or `repr(C, int)`, this will consistently return +/// whether the enum contains any padding bytes. +/// +/// [1]: https://doc.rust-lang.org/1.81.0/reference/type-layout.html#reprc-enums-with-fields +/// [2]: https://doc.rust-lang.org/1.81.0/reference/type-layout.html#primitive-representation-of-enums-with-fields +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! enum_padding { + ($t:ty, $_align:expr, $packed:expr, $disc:ty, $([$($ts:ty),*]),*) => {{ + // The `align` and `packed` directives are irrelevant. `$align` can be + // ignored because regardless of if and how it is set, comparing the + // size of `$t` to each of its field sizes is a reliable indicator of + // the presence of padding. `$packed` is irrelevant because it is + // forbidden on enums. + #[allow(clippy::as_conversions)] + const _: [(); 1] = [(); $packed.is_none() as usize]; + let mut max = 0; + $({ + let padding = $crate::util::macro_util::size_of::<$t>() + - ( + $crate::util::macro_util::size_of::<$disc>() + $(+ $crate::util::macro_util::size_of::<$ts>())* + ); + if padding > max { + max = padding; + } + })* + max + }}; +} + +/// Unwraps an infallible `Result`. +#[doc(hidden)] +#[macro_export] +macro_rules! into_inner { + ($e:expr) => { + match $e { + $crate::util::macro_util::core_reexport::result::Result::Ok(e) => e, + $crate::util::macro_util::core_reexport::result::Result::Err(i) => match i {}, + } + }; +} + +/// Translates an identifier or tuple index into a numeric identifier. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! ident_id { + ($field:ident) => { + $crate::util::macro_util::hash_name(stringify!($field)) + }; + ($field:literal) => { + $field + }; +} + +/// Computes the hash of a string. +/// +/// NOTE(#2749) on hash collisions: This function's output only needs to be +/// deterministic within a particular compilation. Thus, if a user ever reports +/// a hash collision (very unlikely given the <= 16-byte special case), we can +/// strengthen the hash function at that point and publish a new version. Since +/// this is computed at compile time on small strings, we can easily use more +/// expensive and higher-quality hash functions if need be. +#[inline(always)] +#[must_use] +#[allow(clippy::as_conversions, clippy::indexing_slicing, clippy::arithmetic_side_effects)] +pub const fn hash_name(name: &str) -> i128 { + let name = name.as_bytes(); + + // We guarantee freedom from hash collisions between any two strings of + // length 16 or less by having the hashes of such strings be equal to + // their value. There is still a possibility that such strings will have + // the same value as the hash of a string of length > 16. + if name.len() <= size_of::() { + let mut bytes = [0u8; 16]; + + let mut i = 0; + while i < name.len() { + bytes[i] = name[i]; + i += 1; + } + + return i128::from_ne_bytes(bytes); + }; + + // An implementation of FxHasher, although returning a u128. Probably + // not as strong as it could be, but probably more collision resistant + // than normal 64-bit FxHasher. + let mut hash = 0u128; + let mut i = 0; + while i < name.len() { + // This is just FxHasher's `0x517cc1b727220a95` constant + // concatenated back-to-back. + const K: u128 = 0x517cc1b727220a95517cc1b727220a95; + hash = (hash.rotate_left(5) ^ (name[i] as u128)).wrapping_mul(K); + i += 1; + } + i128::from_ne_bytes(hash.to_ne_bytes()) +} + +/// Attempts to transmute `Src` into `Dst`. +/// +/// A helper for `try_transmute!`. +/// +/// # Panics +/// +/// `try_transmute` may either produce a post-monomorphization error or a panic +/// if `Dst` is bigger than `Src`. Otherwise, `try_transmute` panics under the +/// same circumstances as [`is_bit_valid`]. +/// +/// [`is_bit_valid`]: TryFromBytes::is_bit_valid +#[inline(always)] +pub fn try_transmute(src: Src) -> Result> +where + Src: IntoBytes, + Dst: TryFromBytes, +{ + static_assert!(Src, Dst => mem::size_of::() == mem::size_of::()); + + let mu_src = mem::MaybeUninit::new(src); + // SAFETY: `MaybeUninit` has no validity requirements. + let mu_dst: mem::MaybeUninit> = + unsafe { crate::util::transmute_unchecked(mu_src) }; + + let ptr = Ptr::from_ref(&mu_dst); + + // SAFETY: Since `Src: IntoBytes`, and since `size_of::() == + // size_of::()` by the preceding assertion, all of `mu_dst`'s bytes are + // initialized. `MaybeUninit` has no validity requirements, so even if + // `ptr` is used to mutate its referent (which it actually can't be - it's + // a shared `ReadOnly` pointer), that won't violate its referent's validity. + let ptr = unsafe { ptr.assume_validity::() }; + if Dst::is_bit_valid(ptr.cast::<_, CastSized, _>()) { + // SAFETY: Since `Dst::is_bit_valid`, we know that `ptr`'s referent is + // bit-valid for `Dst`. `ptr` points to `mu_dst`, and no intervening + // operations have mutated it, so it is a bit-valid `Dst`. + Ok(ReadOnly::into_inner(unsafe { mu_dst.assume_init() })) + } else { + // SAFETY: `MaybeUninit` has no validity requirements. + let mu_src: mem::MaybeUninit = unsafe { crate::util::transmute_unchecked(mu_dst) }; + // SAFETY: `mu_dst`/`mu_src` was constructed from `src` and never + // modified, so it is still bit-valid. + Err(ValidityError::new(unsafe { mu_src.assume_init() })) + } +} + +/// See `try_transmute_ref!` documentation. +pub trait TryTransmuteRefDst<'a> { + type Dst: ?Sized; + + /// See `try_transmute_ref!` documentation. + fn try_transmute_ref(self) -> Result<&'a Self::Dst, ValidityError<&'a Self::Src, Self::Dst>> + where + Self: TryTransmuteRefSrc<'a>, + Self::Src: IntoBytes + Immutable + KnownLayout, + Self::Dst: TryFromBytes + Immutable + KnownLayout; +} + +pub trait TryTransmuteRefSrc<'a> { + type Src: ?Sized; +} + +impl<'a, Src, Dst> TryTransmuteRefSrc<'a> for Wrap<&'a Src, &'a Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + type Src = Src; +} + +impl<'a, Src, Dst> TryTransmuteRefDst<'a> for Wrap<&'a Src, &'a Dst> +where + Src: IntoBytes + Immutable + KnownLayout + ?Sized, + Dst: TryFromBytes + Immutable + KnownLayout + ?Sized, +{ + type Dst = Dst; + + #[inline(always)] + fn try_transmute_ref( + self, + ) -> Result< + &'a Dst, + ValidityError<&'a as TryTransmuteRefSrc<'a>>::Src, Dst>, + > { + let ptr = Ptr::from_ref(self.0); + #[rustfmt::skip] + let res = ptr.try_with(#[inline(always)] |ptr| { + let ptr = ptr.recall_validity::(); + let ptr = ptr.cast::<_, crate::layout::CastFrom, _>(); + ptr.try_into_valid() + }); + match res { + Ok(ptr) => { + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + // SAFETY: We have checked that `Dst` does not have a stricter + // alignment requirement than `Src`. + let ptr = unsafe { ptr.assume_alignment::() }; + Ok(ptr.as_ref()) + } + Err(err) => Err(err.map_src(Ptr::as_ref)), + } + } +} + +pub trait TryTransmuteMutDst<'a> { + type Dst: ?Sized; + + /// See `try_transmute_mut!` documentation. + fn try_transmute_mut( + self, + ) -> Result<&'a mut Self::Dst, ValidityError<&'a mut Self::Src, Self::Dst>> + where + Self: TryTransmuteMutSrc<'a>, + Self::Src: IntoBytes, + Self::Dst: TryFromBytes; +} + +pub trait TryTransmuteMutSrc<'a> { + type Src: ?Sized; +} + +impl<'a, Src, Dst> TryTransmuteMutSrc<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + type Src = Src; +} + +impl<'a, Src, Dst> TryTransmuteMutDst<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: FromBytes + IntoBytes + KnownLayout + ?Sized, + Dst: TryFromBytes + IntoBytes + KnownLayout + ?Sized, +{ + type Dst = Dst; + + #[inline(always)] + fn try_transmute_mut( + self, + ) -> Result< + &'a mut Dst, + ValidityError<&'a mut as TryTransmuteMutSrc<'a>>::Src, Dst>, + > { + let ptr = Ptr::from_mut(self.0); + // SAFETY: The provided closure returns the only copy of `ptr`. + #[rustfmt::skip] + let res = unsafe { + ptr.try_with_unchecked(#[inline(always)] |ptr| { + let ptr = ptr.recall_validity::(); + let ptr = ptr.cast::<_, crate::layout::CastFrom, _>(); + ptr.try_into_valid() + }) + }; + match res { + Ok(ptr) => { + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + // SAFETY: We have checked that `Dst` does not have a stricter + // alignment requirement than `Src`. + let ptr = unsafe { ptr.assume_alignment::() }; + Ok(ptr.as_mut()) + } + Err(err) => Err(err.map_src(Ptr::as_mut)), + } + } +} + +// Used in `transmute_ref!` and friends. +// +// This permits us to use the autoref specialization trick to dispatch to +// associated functions for `transmute_ref` and `transmute_mut` when both `Src` +// and `Dst` are `Sized`, and to trait methods otherwise. The associated +// functions, unlike the trait methods, do not require a `KnownLayout` bound. +// This permits us to add support for transmuting references to unsized types +// without breaking backwards-compatibility (on v0.8.x) with the old +// implementation, which did not require a `KnownLayout` bound to transmute +// sized types. +#[derive(Copy, Clone)] +pub struct Wrap(pub Src, pub PhantomData); + +impl Wrap { + #[inline(always)] + pub const fn new(src: Src) -> Self { + Wrap(src, PhantomData) + } +} + +impl<'a, Src, Dst> Wrap<&'a Src, &'a Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + #[allow(clippy::must_use_candidate, clippy::missing_inline_in_public_items, clippy::empty_loop)] + pub const fn transmute_ref_inference_helper(self) -> &'a Dst { + loop {} + } +} + +impl<'a, Src, Dst> Wrap<&'a Src, &'a Dst> { + /// # Safety + /// The caller must guarantee that: + /// - `Src: IntoBytes + Immutable` + /// - `Dst: FromBytes + Immutable` + /// + /// # PME + /// + /// Instantiating this method PMEs unless both: + /// - `mem::size_of::() == mem::size_of::()` + /// - `mem::align_of::() <= mem::align_of::()` + #[inline(always)] + #[must_use] + pub const unsafe fn transmute_ref(self) -> &'a Dst { + static_assert!(Src, Dst => mem::size_of::() == mem::size_of::()); + static_assert!(Src, Dst => mem::align_of::() <= mem::align_of::()); + + let src: *const Src = self.0; + let dst = src.cast::(); + // SAFETY: + // - We know that it is sound to view the target type of the input + // reference (`Src`) as the target type of the output reference + // (`Dst`) because the caller has guaranteed that `Src: IntoBytes`, + // `Dst: FromBytes`, and `size_of::() == size_of::()`. + // - We know that there are no `UnsafeCell`s, and thus we don't have to + // worry about `UnsafeCell` overlap, because `Src: Immutable` and + // `Dst: Immutable`. + // - The caller has guaranteed that alignment is not increased. + // - We know that the returned lifetime will not outlive the input + // lifetime thanks to the lifetime bounds on this function. + // + // FIXME(#67): Once our MSRV is 1.58, replace this `transmute` with + // `&*dst`. + #[allow(clippy::transmute_ptr_to_ref)] + unsafe { + mem::transmute(dst) + } + } + + #[inline(always)] + pub fn try_transmute_ref(self) -> Result<&'a Dst, ValidityError<&'a Src, Dst>> + where + Src: IntoBytes + Immutable, + Dst: TryFromBytes + Immutable, + { + static_assert!(Src => mem::align_of::() == mem::align_of::>()); + static_assert!(Dst => mem::align_of::() == mem::align_of::>()); + + // SAFETY: By the preceding assert, `Src` and `Wrapping` have the + // same alignment. + let src: &Wrapping = + unsafe { crate::util::transmute_ref::<_, _, BecauseImmutable>(self.0) }; + let src = Wrap::new(src); + , &'a Wrapping> as TryTransmuteRefDst<'a>>::try_transmute_ref( + src, + ) + .map( + // SAFETY: By the preceding assert, `Dst` and `Wrapping` have + // the same alignment. + #[inline(always)] + |dst| unsafe { crate::util::transmute_ref::<_, _, BecauseImmutable>(dst) }, + ) + .map_err( + #[inline(always)] + |err| { + // SAFETY: By the preceding assert, `Src` and `Wrapping` have the + // same alignment. + ValidityError::new(unsafe { + crate::util::transmute_ref::<_, _, BecauseImmutable>(err.into_src()) + }) + }, + ) + } +} + +impl<'a, Src, Dst> Wrap<&'a mut Src, &'a mut Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + #[allow(clippy::must_use_candidate, clippy::missing_inline_in_public_items, clippy::empty_loop)] + pub fn transmute_mut_inference_helper(self) -> &'a mut Dst { + loop {} + } +} + +impl<'a, Src, Dst> Wrap<&'a mut Src, &'a mut Dst> { + /// Transmutes a mutable reference of one type to a mutable reference of + /// another type. + /// + /// # PME + /// + /// Instantiating this method PMEs unless both: + /// - `mem::size_of::() == mem::size_of::()` + /// - `mem::align_of::() <= mem::align_of::()` + #[inline(always)] + #[must_use] + pub fn transmute_mut(self) -> &'a mut Dst + where + Src: FromBytes + IntoBytes, + Dst: FromBytes + IntoBytes, + { + static_assert!(Src, Dst => mem::size_of::() == mem::size_of::()); + static_assert!(Src, Dst => mem::align_of::() <= mem::align_of::()); + + let src: *mut Src = self.0; + let dst = src.cast::(); + // SAFETY: + // - We know that it is sound to view the target type of the input + // reference (`Src`) as the target type of the output reference + // (`Dst`) and vice-versa because `Src: FromBytes + IntoBytes`, `Dst: + // FromBytes + IntoBytes`, and (as asserted above) `size_of::() + // == size_of::()`. + // - We asserted above that alignment will not increase. + // - We know that the returned lifetime will not outlive the input + // lifetime thanks to the lifetime bounds on this function. + unsafe { &mut *dst } + } + + #[inline(always)] + pub fn try_transmute_mut(self) -> Result<&'a mut Dst, ValidityError<&'a mut Src, Dst>> + where + Src: FromBytes + IntoBytes, + Dst: TryFromBytes + IntoBytes, + { + static_assert!(Src => mem::align_of::() == mem::align_of::>()); + static_assert!(Dst => mem::align_of::() == mem::align_of::>()); + + // SAFETY: By the preceding assert, `Src` and `Wrapping` have the + // same alignment. + let src: &mut Wrapping = + unsafe { crate::util::transmute_mut::<_, _, (_, (_, _))>(self.0) }; + let src = Wrap::new(src); + , &'a mut Wrapping> as TryTransmuteMutDst<'a>> + ::try_transmute_mut(src) + // SAFETY: By the preceding assert, `Dst` and `Wrapping` have the + // same alignment. + .map(|dst| unsafe { crate::util::transmute_mut::<_, _, (_, (_, _))>(dst) }) + .map_err(|err| { + // SAFETY: By the preceding assert, `Src` and `Wrapping` have the + // same alignment. + ValidityError::new(unsafe { + crate::util::transmute_mut::<_, _, (_, (_, _))>(err.into_src()) + }) + }) + } +} + +pub trait TransmuteRefDst<'a> { + type Dst: ?Sized; + + #[must_use] + fn transmute_ref(self) -> &'a Self::Dst; +} + +impl<'a, Src: ?Sized, Dst: ?Sized> TransmuteRefDst<'a> for Wrap<&'a Src, &'a Dst> +where + Src: KnownLayout + IntoBytes + Immutable, + Dst: KnownLayout + FromBytes + Immutable, +{ + type Dst = Dst; + + #[inline(always)] + fn transmute_ref(self) -> &'a Dst { + let ptr = Ptr::from_ref(self.0) + .recall_validity::() + .transmute_with::, (crate::pointer::BecauseMutationCompatible, _)>() + .recall_validity::(); + + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + + // SAFETY: The preceding `static_assert!` ensures that + // `Src::LAYOUT.align >= Dst::LAYOUT.align`. Since `self` is + // validly-aligned for `Src`, it is also validly-aligned for `Dst`. + let ptr = unsafe { ptr.assume_alignment() }; + + ptr.as_ref() + } +} + +pub trait TransmuteMutDst<'a> { + type Dst: ?Sized; + #[must_use] + fn transmute_mut(self) -> &'a mut Self::Dst; +} + +impl<'a, Src: ?Sized, Dst: ?Sized> TransmuteMutDst<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: KnownLayout + FromBytes + IntoBytes, + Dst: KnownLayout + FromBytes + IntoBytes, +{ + type Dst = Dst; + + #[inline(always)] + fn transmute_mut(self) -> &'a mut Dst { + let ptr = Ptr::from_mut(self.0) + .recall_validity::() + .transmute_with::, _>() + .recall_validity::(); + + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + + // SAFETY: The preceding `static_assert!` ensures that + // `Src::LAYOUT.align >= Dst::LAYOUT.align`. Since `self` is + // validly-aligned for `Src`, it is also validly-aligned for `Dst`. + let ptr = unsafe { ptr.assume_alignment() }; + + ptr.as_mut() + } +} + +/// A function which emits a warning if its return value is not used. +#[must_use] +#[inline(always)] +pub const fn must_use(t: T) -> T { + t +} + +// NOTE: We can't change this to a `pub use core as core_reexport` until [1] is +// fixed or we update to a semver-breaking version (as of this writing, 0.8.0) +// on the `main` branch. +// +// [1] https://github.com/obi1kenobi/cargo-semver-checks/issues/573 +pub mod core_reexport { + pub use core::*; + + pub mod mem { + pub use core::mem::*; + } +} + +#[cfg(test)] +mod tests { + use core::num::NonZeroUsize; + + use crate::util::testutil::*; + + #[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] + mod nightly { + use super::super::*; + use crate::util::testutil::*; + + // FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): + // Remove this `cfg` when `size_of_val_raw` is stabilized. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_trailing_field_offset() { + assert_eq!(mem::align_of::(), _64K); + + macro_rules! test { + (#[$cfg:meta] ($($ts:ty),* ; $trailing_field_ty:ty) => $expect:expr) => {{ + #[$cfg] + struct Test($(#[allow(dead_code)] $ts,)* #[allow(dead_code)] $trailing_field_ty); + assert_eq!(test!(@offset $($ts),* ; $trailing_field_ty), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* ($($ts:ty),* ; $trailing_field_ty:ty) => $expect:expr) => { + test!(#[$cfg] ($($ts),* ; $trailing_field_ty) => $expect); + test!($(#[$cfgs])* ($($ts),* ; $trailing_field_ty) => $expect); + }; + (@offset ; $_trailing:ty) => { trailing_field_offset!(Test, 0) }; + (@offset $_t:ty ; $_trailing:ty) => { trailing_field_offset!(Test, 1) }; + } + + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)](; u8) => Some(0)); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)](; [u8]) => Some(0)); + test!(#[repr(C)] #[repr(C, packed)] (u8; u8) => Some(1)); + test!(#[repr(C)] (; AU64) => Some(0)); + test!(#[repr(C)] (; [AU64]) => Some(0)); + test!(#[repr(C)] (u8; AU64) => Some(8)); + test!(#[repr(C)] (u8; [AU64]) => Some(8)); + + #[derive( + Immutable, FromBytes, Eq, PartialEq, Ord, PartialOrd, Default, Debug, Copy, Clone, + )] + #[repr(C)] + pub(crate) struct Nested { + _t: T, + _u: U, + } + + test!(#[repr(C)] (; Nested) => Some(0)); + test!(#[repr(C)] (; Nested) => Some(0)); + test!(#[repr(C)] (u8; Nested) => Some(8)); + test!(#[repr(C)] (u8; Nested) => Some(8)); + + // Test that `packed(N)` limits the offset of the trailing field. + test!(#[repr(C, packed( 1))] (u8; elain::Align< 2>) => Some( 1)); + test!(#[repr(C, packed( 2))] (u8; elain::Align< 4>) => Some( 2)); + test!(#[repr(C, packed( 4))] (u8; elain::Align< 8>) => Some( 4)); + test!(#[repr(C, packed( 8))] (u8; elain::Align< 16>) => Some( 8)); + test!(#[repr(C, packed( 16))] (u8; elain::Align< 32>) => Some( 16)); + test!(#[repr(C, packed( 32))] (u8; elain::Align< 64>) => Some( 32)); + test!(#[repr(C, packed( 64))] (u8; elain::Align< 128>) => Some( 64)); + test!(#[repr(C, packed( 128))] (u8; elain::Align< 256>) => Some( 128)); + test!(#[repr(C, packed( 256))] (u8; elain::Align< 512>) => Some( 256)); + test!(#[repr(C, packed( 512))] (u8; elain::Align< 1024>) => Some( 512)); + test!(#[repr(C, packed( 1024))] (u8; elain::Align< 2048>) => Some( 1024)); + test!(#[repr(C, packed( 2048))] (u8; elain::Align< 4096>) => Some( 2048)); + test!(#[repr(C, packed( 4096))] (u8; elain::Align< 8192>) => Some( 4096)); + test!(#[repr(C, packed( 8192))] (u8; elain::Align< 16384>) => Some( 8192)); + test!(#[repr(C, packed( 16384))] (u8; elain::Align< 32768>) => Some( 16384)); + test!(#[repr(C, packed( 32768))] (u8; elain::Align< 65536>) => Some( 32768)); + test!(#[repr(C, packed( 65536))] (u8; elain::Align< 131072>) => Some( 65536)); + /* Alignments above 65536 are not yet supported. + test!(#[repr(C, packed( 131072))] (u8; elain::Align< 262144>) => Some( 131072)); + test!(#[repr(C, packed( 262144))] (u8; elain::Align< 524288>) => Some( 262144)); + test!(#[repr(C, packed( 524288))] (u8; elain::Align< 1048576>) => Some( 524288)); + test!(#[repr(C, packed( 1048576))] (u8; elain::Align< 2097152>) => Some( 1048576)); + test!(#[repr(C, packed( 2097152))] (u8; elain::Align< 4194304>) => Some( 2097152)); + test!(#[repr(C, packed( 4194304))] (u8; elain::Align< 8388608>) => Some( 4194304)); + test!(#[repr(C, packed( 8388608))] (u8; elain::Align< 16777216>) => Some( 8388608)); + test!(#[repr(C, packed( 16777216))] (u8; elain::Align< 33554432>) => Some( 16777216)); + test!(#[repr(C, packed( 33554432))] (u8; elain::Align< 67108864>) => Some( 33554432)); + test!(#[repr(C, packed( 67108864))] (u8; elain::Align< 33554432>) => Some( 67108864)); + test!(#[repr(C, packed( 33554432))] (u8; elain::Align<134217728>) => Some( 33554432)); + test!(#[repr(C, packed(134217728))] (u8; elain::Align<268435456>) => Some(134217728)); + test!(#[repr(C, packed(268435456))] (u8; elain::Align<268435456>) => Some(268435456)); + */ + + // Test that `align(N)` does not limit the offset of the trailing field. + test!(#[repr(C, align( 1))] (u8; elain::Align< 2>) => Some( 2)); + test!(#[repr(C, align( 2))] (u8; elain::Align< 4>) => Some( 4)); + test!(#[repr(C, align( 4))] (u8; elain::Align< 8>) => Some( 8)); + test!(#[repr(C, align( 8))] (u8; elain::Align< 16>) => Some( 16)); + test!(#[repr(C, align( 16))] (u8; elain::Align< 32>) => Some( 32)); + test!(#[repr(C, align( 32))] (u8; elain::Align< 64>) => Some( 64)); + test!(#[repr(C, align( 64))] (u8; elain::Align< 128>) => Some( 128)); + test!(#[repr(C, align( 128))] (u8; elain::Align< 256>) => Some( 256)); + test!(#[repr(C, align( 256))] (u8; elain::Align< 512>) => Some( 512)); + test!(#[repr(C, align( 512))] (u8; elain::Align< 1024>) => Some( 1024)); + test!(#[repr(C, align( 1024))] (u8; elain::Align< 2048>) => Some( 2048)); + test!(#[repr(C, align( 2048))] (u8; elain::Align< 4096>) => Some( 4096)); + test!(#[repr(C, align( 4096))] (u8; elain::Align< 8192>) => Some( 8192)); + test!(#[repr(C, align( 8192))] (u8; elain::Align< 16384>) => Some( 16384)); + test!(#[repr(C, align( 16384))] (u8; elain::Align< 32768>) => Some( 32768)); + test!(#[repr(C, align( 32768))] (u8; elain::Align< 65536>) => Some( 65536)); + /* Alignments above 65536 are not yet supported. + test!(#[repr(C, align( 65536))] (u8; elain::Align< 131072>) => Some( 131072)); + test!(#[repr(C, align( 131072))] (u8; elain::Align< 262144>) => Some( 262144)); + test!(#[repr(C, align( 262144))] (u8; elain::Align< 524288>) => Some( 524288)); + test!(#[repr(C, align( 524288))] (u8; elain::Align< 1048576>) => Some( 1048576)); + test!(#[repr(C, align( 1048576))] (u8; elain::Align< 2097152>) => Some( 2097152)); + test!(#[repr(C, align( 2097152))] (u8; elain::Align< 4194304>) => Some( 4194304)); + test!(#[repr(C, align( 4194304))] (u8; elain::Align< 8388608>) => Some( 8388608)); + test!(#[repr(C, align( 8388608))] (u8; elain::Align< 16777216>) => Some( 16777216)); + test!(#[repr(C, align( 16777216))] (u8; elain::Align< 33554432>) => Some( 33554432)); + test!(#[repr(C, align( 33554432))] (u8; elain::Align< 67108864>) => Some( 67108864)); + test!(#[repr(C, align( 67108864))] (u8; elain::Align< 33554432>) => Some( 33554432)); + test!(#[repr(C, align( 33554432))] (u8; elain::Align<134217728>) => Some(134217728)); + test!(#[repr(C, align(134217728))] (u8; elain::Align<268435456>) => Some(268435456)); + */ + } + + // FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): + // Remove this `cfg` when `size_of_val_raw` is stabilized. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_align_of_dst() { + // Test that `align_of!` correctly computes the alignment of DSTs. + assert_eq!(align_of!([elain::Align<1>]), Some(1)); + assert_eq!(align_of!([elain::Align<2>]), Some(2)); + assert_eq!(align_of!([elain::Align<4>]), Some(4)); + assert_eq!(align_of!([elain::Align<8>]), Some(8)); + assert_eq!(align_of!([elain::Align<16>]), Some(16)); + assert_eq!(align_of!([elain::Align<32>]), Some(32)); + assert_eq!(align_of!([elain::Align<64>]), Some(64)); + assert_eq!(align_of!([elain::Align<128>]), Some(128)); + assert_eq!(align_of!([elain::Align<256>]), Some(256)); + assert_eq!(align_of!([elain::Align<512>]), Some(512)); + assert_eq!(align_of!([elain::Align<1024>]), Some(1024)); + assert_eq!(align_of!([elain::Align<2048>]), Some(2048)); + assert_eq!(align_of!([elain::Align<4096>]), Some(4096)); + assert_eq!(align_of!([elain::Align<8192>]), Some(8192)); + assert_eq!(align_of!([elain::Align<16384>]), Some(16384)); + assert_eq!(align_of!([elain::Align<32768>]), Some(32768)); + assert_eq!(align_of!([elain::Align<65536>]), Some(65536)); + /* Alignments above 65536 are not yet supported. + assert_eq!(align_of!([elain::Align<131072>]), Some(131072)); + assert_eq!(align_of!([elain::Align<262144>]), Some(262144)); + assert_eq!(align_of!([elain::Align<524288>]), Some(524288)); + assert_eq!(align_of!([elain::Align<1048576>]), Some(1048576)); + assert_eq!(align_of!([elain::Align<2097152>]), Some(2097152)); + assert_eq!(align_of!([elain::Align<4194304>]), Some(4194304)); + assert_eq!(align_of!([elain::Align<8388608>]), Some(8388608)); + assert_eq!(align_of!([elain::Align<16777216>]), Some(16777216)); + assert_eq!(align_of!([elain::Align<33554432>]), Some(33554432)); + assert_eq!(align_of!([elain::Align<67108864>]), Some(67108864)); + assert_eq!(align_of!([elain::Align<33554432>]), Some(33554432)); + assert_eq!(align_of!([elain::Align<134217728>]), Some(134217728)); + assert_eq!(align_of!([elain::Align<268435456>]), Some(268435456)); + */ + } + } + + #[test] + fn test_enum_casts() { + // Test that casting the variants of enums with signed integer reprs to + // unsigned integers obeys expected signed -> unsigned casting rules. + + #[repr(i8)] + enum ReprI8 { + MinusOne = -1, + Zero = 0, + Min = i8::MIN, + Max = i8::MAX, + } + + #[allow(clippy::as_conversions)] + let x = ReprI8::MinusOne as u8; + assert_eq!(x, u8::MAX); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Zero as u8; + assert_eq!(x, 0); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Min as u8; + assert_eq!(x, 128); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Max as u8; + assert_eq!(x, 127); + } + + #[test] + fn test_struct_padding() { + // Test that, for each provided repr, `struct_padding!` reports the + // expected value. + macro_rules! test { + (#[$cfg:meta] ($($ts:ty),*) => $expect:expr) => {{ + #[$cfg] + #[allow(dead_code)] + struct Test($($ts),*); + assert_eq!(struct_padding!(Test, None::, None::, [$($ts),*]), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* ($($ts:ty),*) => $expect:expr) => { + test!(#[$cfg] ($($ts),*) => $expect); + test!($(#[$cfgs])* ($($ts),*) => $expect); + }; + } + + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] () => 0); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] (u8) => 0); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] (u8, ()) => 0); + test!(#[repr(C)] #[repr(packed)] (u8, u8) => 0); + + test!(#[repr(C)] (u8, AU64) => 7); + // Rust won't let you put `#[repr(packed)]` on a type which contains a + // `#[repr(align(n > 1))]` type (`AU64`), so we have to use `u64` here. + // It's not ideal, but it definitely has align > 1 on /some/ of our CI + // targets, and this isn't a particularly complex macro we're testing + // anyway. + test!(#[repr(packed)] (u8, u64) => 0); + } + + #[test] + fn test_repr_c_struct_padding() { + // Test that, for each provided repr, `repr_c_struct_padding!` reports + // the expected value. + macro_rules! test { + (($($ts:tt),*) => $expect:expr) => {{ + #[repr(C)] + #[allow(dead_code)] + struct Test($($ts),*); + assert_eq!(repr_c_struct_has_padding!(Test, None::, None::, [$($ts),*]), $expect); + }}; + } + + // Test static padding + test!(() => false); + test!(([u8]) => false); + test!((u8) => false); + test!((u8, [u8]) => false); + test!((u8, ()) => false); + test!((u8, (), [u8]) => false); + test!((u8, u8) => false); + test!((u8, u8, [u8]) => false); + + test!((u8, AU64) => true); + test!((u8, AU64, [u8]) => true); + + // Test dynamic padding + test!((AU64, [AU64]) => false); + test!((u8, [AU64]) => true); + + #[repr(align(4))] + struct AU32(#[allow(unused)] u32); + test!((AU64, [AU64]) => false); + test!((AU64, [AU32]) => true); + } + + #[test] + fn test_union_padding() { + // Test that, for each provided repr, `union_padding!` reports the + // expected value. + macro_rules! test { + (#[$cfg:meta] {$($fs:ident: $ts:ty),*} => $expect:expr) => {{ + #[$cfg] + #[allow(unused)] // fields are never read + union Test{ $($fs: $ts),* } + assert_eq!(union_padding!(Test, None::, None::, [$($ts),*]), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* {$($fs:ident: $ts:ty),*} => $expect:expr) => { + test!(#[$cfg] {$($fs: $ts),*} => $expect); + test!($(#[$cfgs])* {$($fs: $ts),*} => $expect); + }; + } + + test!(#[repr(C)] #[repr(packed)] {a: u8} => 0); + test!(#[repr(C)] #[repr(packed)] {a: u8, b: u8} => 0); + + // Rust won't let you put `#[repr(packed)]` on a type which contains a + // `#[repr(align(n > 1))]` type (`AU64`), so we have to use `u64` here. + // It's not ideal, but it definitely has align > 1 on /some/ of our CI + // targets, and this isn't a particularly complex macro we're testing + // anyway. + test!(#[repr(C)] #[repr(packed)] {a: u8, b: u64} => 7); + } + + #[test] + fn test_enum_padding() { + // Test that, for each provided repr, `enum_has_padding!` reports the + // expected value. + macro_rules! test { + (#[repr($disc:ident $(, $c:ident)?)] { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => { + test!(@case #[repr($disc $(, $c)?)] { $($vs ($($ts),*),)* } => $expect); + }; + (#[repr($disc:ident $(, $c:ident)?)] #[$cfg:meta] $(#[$cfgs:meta])* { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => { + test!(@case #[repr($disc $(, $c)?)] #[$cfg] { $($vs ($($ts),*),)* } => $expect); + test!(#[repr($disc $(, $c)?)] $(#[$cfgs])* { $($vs ($($ts),*),)* } => $expect); + }; + (@case #[repr($disc:ident $(, $c:ident)?)] $(#[$cfg:meta])? { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => {{ + #[repr($disc $(, $c)?)] + $(#[$cfg])? + #[allow(unused)] // variants and fields are never used + enum Test { + $($vs ($($ts),*),)* + } + assert_eq!( + enum_padding!(Test, None::, None::, $disc, $([$($ts),*]),*), + $expect + ); + }}; + } + + #[allow(unused)] + #[repr(align(2))] + struct U16(u16); + + #[allow(unused)] + #[repr(align(4))] + struct U32(u32); + + test!(#[repr(u8)] #[repr(C)] { + A(u8), + } => 0); + test!(#[repr(u16)] #[repr(C)] { + A(u8, u8), + B(U16), + } => 0); + test!(#[repr(u32)] #[repr(C)] { + A(u8, u8, u8, u8), + B(U16, u8, u8), + C(u8, u8, U16), + D(U16, U16), + E(U32), + } => 0); + + // `repr(int)` can pack the discriminant more efficiently + test!(#[repr(u8)] { + A(u8, U16), + } => 0); + test!(#[repr(u8)] { + A(u8, U16, U32), + } => 0); + + // `repr(C)` cannot + test!(#[repr(u8, C)] { + A(u8, U16), + } => 2); + test!(#[repr(u8, C)] { + A(u8, u8, u8, U32), + } => 4); + + // And field ordering can always cause problems + test!(#[repr(u8)] #[repr(C)] { + A(U16, u8), + } => 2); + test!(#[repr(u8)] #[repr(C)] { + A(U32, u8, u8, u8), + } => 4); + } +} diff --git a/rust/zerocopy/src/util/macros.rs b/rust/zerocopy/src/util/macros.rs new file mode 100644 index 000000000000..7dca5410c84f --- /dev/null +++ b/rust/zerocopy/src/util/macros.rs @@ -0,0 +1,1065 @@ +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +/// Unsafely implements trait(s) for a type. +/// +/// # Safety +/// +/// The trait impl must be sound. +/// +/// When implementing `TryFromBytes`: +/// - If no `is_bit_valid` impl is provided, then it must be valid for +/// `is_bit_valid` to unconditionally return `true`. In other words, it must +/// be the case that any initialized sequence of bytes constitutes a valid +/// instance of `$ty`. +/// - If an `is_bit_valid` impl is provided, then the impl of `is_bit_valid` +/// must only return `true` if its argument refers to a valid `$ty`. +macro_rules! unsafe_impl { + // Implement `$trait` for `$ty` with no bounds. + ($(#[$attr:meta])* $ty:ty: $trait:ident $(; |$candidate:ident| $is_bit_valid:expr)?) => {{ + crate::util::macros::__unsafe(); + + $(#[$attr])* + // SAFETY: The caller promises that this is sound. + unsafe impl $trait for $ty { + unsafe_impl!(@method $trait $(; |$candidate| $is_bit_valid)?); + } + }}; + + // Implement all `$traits` for `$ty` with no bounds. + // + // The 2 arms under this one are there so we can apply + // N attributes for each one of M trait implementations. + // The simple solution of: + // + // ($(#[$attrs:meta])* $ty:ty: $($traits:ident),*) => { + // $( unsafe_impl!( $(#[$attrs])* $ty: $traits ) );* + // } + // + // Won't work. The macro processor sees that the outer repetition + // contains both $attrs and $traits and expects them to match the same + // amount of fragments. + // + // To solve this we must: + // 1. Pack the attributes into a single token tree fragment we can match over. + // 2. Expand the traits. + // 3. Unpack and expand the attributes. + ($(#[$attrs:meta])* $ty:ty: $($traits:ident),*) => { + unsafe_impl!(@impl_traits_with_packed_attrs { $(#[$attrs])* } $ty: $($traits),*) + }; + + (@impl_traits_with_packed_attrs $attrs:tt $ty:ty: $($traits:ident),*) => {{ + $( unsafe_impl!(@unpack_attrs $attrs $ty: $traits); )* + }}; + + (@unpack_attrs { $(#[$attrs:meta])* } $ty:ty: $traits:ident) => { + unsafe_impl!($(#[$attrs])* $ty: $traits); + }; + + // This arm is identical to the following one, except it contains a + // preceding `const`. If we attempt to handle these with a single arm, there + // is an inherent ambiguity between `const` (the keyword) and `const` (the + // ident match for `$tyvar:ident`). + // + // To explain how this works, consider the following invocation: + // + // unsafe_impl!(const N: usize, T: ?Sized + Copy => Clone for Foo); + // + // In this invocation, here are the assignments to meta-variables: + // + // |---------------|------------| + // | Meta-variable | Assignment | + // |---------------|------------| + // | $constname | N | + // | $constty | usize | + // | $tyvar | T | + // | $optbound | Sized | + // | $bound | Copy | + // | $trait | Clone | + // | $ty | Foo | + // |---------------|------------| + // + // The following arm has the same behavior with the exception of the lack of + // support for a leading `const` parameter. + ( + $(#[$attr:meta])* + const $constname:ident : $constty:ident $(,)? + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl!( + @inner + $(#[$attr])* + @const $constname: $constty, + $($tyvar $(: $(? $optbound +)* + $($bound +)*)?,)* + => $trait for $ty $(; |$candidate| $is_bit_valid)? + ); + }; + ( + $(#[$attr:meta])* + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => {{ + unsafe_impl!( + @inner + $(#[$attr])* + $($tyvar $(: $(? $optbound +)* + $($bound +)*)?,)* + => $trait for $ty $(; |$candidate| $is_bit_valid)? + ); + }}; + ( + @inner + $(#[$attr:meta])* + $(@const $constname:ident : $constty:ident,)* + $($tyvar:ident $(: $(? $optbound:ident +)* + $($bound:ident +)* )?,)* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => {{ + crate::util::macros::__unsafe(); + + $(#[$attr])* + #[allow(non_local_definitions)] + // SAFETY: The caller promises that this is sound. + unsafe impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?),* $(, const $constname: $constty,)*> $trait for $ty { + unsafe_impl!(@method $trait $(; |$candidate| $is_bit_valid)?); + } + }}; + + (@method TryFromBytes ; |$candidate:ident| $is_bit_valid:expr) => { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + + #[inline] + fn is_bit_valid($candidate: Maybe<'_, Self, Alignment>) -> bool + where + Alignment: crate::invariant::Alignment, + { + $is_bit_valid + } + }; + (@method TryFromBytes) => { + #[allow(clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + #[inline(always)] + fn is_bit_valid(_candidate: Maybe<'_, Self, Alignment>) -> bool + where + Alignment: crate::invariant::Alignment, + { + true + } + }; + (@method $trait:ident) => { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + }; + (@method $trait:ident; |$_candidate:ident| $_is_bit_valid:expr) => { + compile_error!("Can't provide `is_bit_valid` impl for trait other than `TryFromBytes`"); + }; +} + +/// Implements `$trait` for `$ty` where `$ty: TransmuteFrom<$repr>` (and +/// vice-versa). +/// +/// Calling this macro is safe; the internals of the macro emit appropriate +/// trait bounds which ensure that the given impl is sound. +macro_rules! impl_for_transmute_from { + ( + $(#[$attr:meta])* + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?)? + => $trait:ident for $ty:ty [$repr:ty] + ) => { + const _: () = { + $(#[$attr])* + #[allow(non_local_definitions)] + + // SAFETY: `is_trait` (defined and used below) requires `T: + // TransmuteFrom`, `R: TransmuteFrom`, and `R: $trait`. It is + // called using `$ty` and `$repr`, ensuring that `$ty` and `$repr` + // have equivalent bit validity, and ensuring that `$repr: $trait`. + // The supported traits - `TryFromBytes`, `FromZeros`, `FromBytes`, + // and `IntoBytes` - are defined only in terms of the bit validity + // of a type. Therefore, `$repr: $trait` ensures that `$ty: $trait` + // is sound. + unsafe impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?> $trait for $ty { + #[allow(dead_code, clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() { + use crate::pointer::{*, invariant::Valid}; + + impl_for_transmute_from!(@assert_is_supported_trait $trait); + + fn is_trait() + where + T: TransmuteFrom + ?Sized, + R: TransmuteFrom + ?Sized, + R: $trait, + { + } + + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn f<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?>() { + is_trait::<$ty, $repr>(); + } + } + + impl_for_transmute_from!( + @is_bit_valid + $(<$tyvar $(: $(? $optbound +)* $($bound +)*)?>)? + $trait for $ty [$repr] + ); + } + }; + }; + (@assert_is_supported_trait TryFromBytes) => {}; + (@assert_is_supported_trait FromZeros) => {}; + (@assert_is_supported_trait FromBytes) => {}; + (@assert_is_supported_trait IntoBytes) => {}; + ( + @is_bit_valid + $(<$tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?>)? + TryFromBytes for $ty:ty [$repr:ty] + ) => { + #[inline(always)] + fn is_bit_valid(candidate: $crate::Maybe<'_, Self, Alignment>) -> bool + where + Alignment: $crate::invariant::Alignment, + { + // SAFETY: This macro ensures that `$repr` and `Self` have the same + // size and bit validity. Thus, a bit-valid instance of `$repr` is + // also a bit-valid instance of `Self`. + <$repr as TryFromBytes>::is_bit_valid(candidate.transmute::<_, _, BecauseImmutable>()) + } + }; + ( + @is_bit_valid + $(<$tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?>)? + $trait:ident for $ty:ty [$repr:ty] + ) => { + // Trait other than `TryFromBytes`; no `is_bit_valid` impl. + }; +} + +/// Implements a trait for a type, bounding on each member of the power set of +/// a set of type variables. This is useful for implementing traits for tuples +/// or `fn` types. +/// +/// The last argument is the name of a macro which will be called in every +/// `impl` block, and is expected to expand to the name of the type for which to +/// implement the trait. +/// +/// For example, the invocation: +/// ```ignore +/// unsafe_impl_for_power_set!(A, B => Foo for type!(...)) +/// ``` +/// ...expands to: +/// ```ignore +/// unsafe impl Foo for type!() { ... } +/// unsafe impl Foo for type!(B) { ... } +/// unsafe impl Foo for type!(A, B) { ... } +/// ``` +macro_rules! unsafe_impl_for_power_set { + ( + $first:ident $(, $rest:ident)* $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl_for_power_set!( + $($rest),* $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + unsafe_impl_for_power_set!( + @impl $first $(, $rest)* $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + }; + ( + $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl_for_power_set!( + @impl $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + }; + ( + @impl $($vars:ident),* $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl!( + $($vars,)* $($ret)? => $trait for $macro!($($vars),* $(-> $ret)?) + $(; |$candidate| $is_bit_valid)? + ); + }; +} + +/// Expands to an `Option` type with the given argument types and +/// return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_extern_c_fn { + ($($args:ident),* -> $ret:ident) => { Option $ret> }; +} + +/// Expands to an `Option` type with the given argument +/// types and return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_unsafe_extern_c_fn { + ($($args:ident),* -> $ret:ident) => { Option $ret> }; +} + +/// Expands to an `Option` type with the given argument types and return +/// type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_fn { + ($($args:ident),* -> $ret:ident) => { Option $ret> }; +} + +/// Expands to an `Option` type with the given argument types and +/// return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_unsafe_fn { + ($($args:ident),* -> $ret:ident) => { Option $ret> }; +} + +// This `allow` is needed because, when testing, we export this macro so it can +// be used in `doctests`. +#[allow(rustdoc::private_intra_doc_links)] +/// Implements trait(s) for a type or verifies the given implementation by +/// referencing an existing (derived) implementation. +/// +/// This macro exists so that we can provide zerocopy-derive as an optional +/// dependency and still get the benefit of using its derives to validate that +/// our trait impls are sound. +/// +/// When compiling without `--cfg 'feature = "derive"` and without `--cfg test`, +/// `impl_or_verify!` emits the provided trait impl. When compiling with either +/// of those cfgs, it is expected that the type in question is deriving the +/// traits instead. In this case, `impl_or_verify!` emits code which validates +/// that the given trait impl is at least as restrictive as the the impl emitted +/// by the custom derive. This has the effect of confirming that the impl which +/// is emitted when the `derive` feature is disabled is actually sound (on the +/// assumption that the impl emitted by the custom derive is sound). +/// +/// The caller is still required to provide a safety comment (e.g. using the +/// `const _: () = unsafe` macro). The reason for this restriction is that, +/// while `impl_or_verify!` can guarantee that the provided impl is sound when +/// it is compiled with the appropriate cfgs, there is no way to guarantee that +/// it is ever compiled with those cfgs. In particular, it would be possible to +/// accidentally place an `impl_or_verify!` call in a context that is only ever +/// compiled when the `derive` feature is disabled. If that were to happen, +/// there would be nothing to prevent an unsound trait impl from being emitted. +/// Requiring a safety comment reduces the likelihood of emitting an unsound +/// impl in this case, and also provides useful documentation for readers of the +/// code. +/// +/// Finally, if a `TryFromBytes::is_bit_valid` impl is provided, it must adhere +/// to the safety preconditions of [`unsafe_impl!`]. +/// +/// ## Example +/// +/// ```rust,ignore +/// // Note that these derives are gated by `feature = "derive"` +/// #[cfg_attr(any(feature = "derive", test), derive(FromZeros, FromBytes, IntoBytes, Unaligned))] +/// #[repr(transparent)] +/// struct Wrapper(T); +/// +/// const _: () = unsafe { +/// /// SAFETY: +/// /// `Wrapper` is `repr(transparent)`, so it is sound to implement any +/// /// zerocopy trait if `T` implements that trait. +/// impl_or_verify!(T: FromZeros => FromZeros for Wrapper); +/// impl_or_verify!(T: FromBytes => FromBytes for Wrapper); +/// impl_or_verify!(T: IntoBytes => IntoBytes for Wrapper); +/// impl_or_verify!(T: Unaligned => Unaligned for Wrapper); +/// } +/// ``` +#[cfg_attr(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE, macro_export)] // Used in `doctests.rs` +#[doc(hidden)] +macro_rules! impl_or_verify { + // The following two match arms follow the same pattern as their + // counterparts in `unsafe_impl!`; see the documentation on those arms for + // more details. + ( + const $constname:ident : $constty:ident $(,)? + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty + ) => { + impl_or_verify!(@impl { unsafe_impl!( + const $constname: $constty, $($tyvar $(: $(? $optbound +)* $($bound +)*)?),* => $trait for $ty + ); }); + impl_or_verify!(@verify $trait, { + impl Subtrait for $ty {} + }); + }; + ( + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + impl_or_verify!(@impl { unsafe_impl!( + $($tyvar $(: $(? $optbound +)* $($bound +)*)?),* => $trait for $ty + $(; |$candidate| $is_bit_valid)? + ); }); + impl_or_verify!(@verify $trait, { + impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?),*> Subtrait for $ty {} + }); + }; + (@impl $impl_block:tt) => { + #[cfg(not(any(feature = "derive", test)))] + { $impl_block }; + }; + (@verify $trait:ident, $impl_block:tt) => { + #[cfg(any(feature = "derive", test))] + { + // On some toolchains, `Subtrait` triggers the `dead_code` lint + // because it is implemented but never used. + #[allow(dead_code)] + trait Subtrait: $trait {} + $impl_block + }; + }; +} + +/// Implements `KnownLayout` for a sized type. +macro_rules! impl_known_layout { + ($(const $constvar:ident : $constty:ty, $tyvar:ident $(: ?$optbound:ident)? => $ty:ty),* $(,)?) => { + $(impl_known_layout!(@inner const $constvar: $constty, $tyvar $(: ?$optbound)? => $ty);)* + }; + ($($tyvar:ident $(: ?$optbound:ident)? => $ty:ty),* $(,)?) => { + $(impl_known_layout!(@inner , $tyvar $(: ?$optbound)? => $ty);)* + }; + ($($(#[$attrs:meta])* $ty:ty),*) => { $(impl_known_layout!(@inner , => $(#[$attrs])* $ty);)* }; + (@inner $(const $constvar:ident : $constty:ty)? , $($tyvar:ident $(: ?$optbound:ident)?)? => $(#[$attrs:meta])* $ty:ty) => { + const _: () = { + use core::ptr::NonNull; + + #[allow(non_local_definitions)] + $(#[$attrs])* + // SAFETY: Delegates safety to `DstLayout::for_type`. + unsafe impl<$($tyvar $(: ?$optbound)?)? $(, const $constvar : $constty)?> KnownLayout for $ty { + #[allow(clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() where Self: Sized {} + + type PointerMetadata = (); + + // SAFETY: `CoreMaybeUninit::LAYOUT` and `T::LAYOUT` are + // identical because `CoreMaybeUninit` has the same size and + // alignment as `T` [1], and `CoreMaybeUninit` admits + // uninitialized bytes in all positions. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit` is guaranteed to have the same size, + // alignment, and ABI as `T` + type MaybeUninit = core::mem::MaybeUninit; + + const LAYOUT: crate::DstLayout = crate::DstLayout::for_type::<$ty>(); + + // SAFETY: `.cast` preserves address and provenance. + // + // FIXME(#429): Add documentation to `.cast` that promises that + // it preserves provenance. + #[inline(always)] + fn raw_from_ptr_len(bytes: NonNull, _meta: ()) -> NonNull { + bytes.cast::() + } + + #[inline(always)] + fn pointer_to_metadata(_ptr: *mut Self) -> () { + } + } + }; + }; +} + +/// Implements `KnownLayout` for a type in terms of the implementation of +/// another type with the same representation. +/// +/// # Safety +/// +/// - `$ty` and `$repr` must have the same: +/// - Fixed prefix size +/// - Alignment +/// - (For DSTs) trailing slice element size +/// - It must be valid to perform an `as` cast from `*mut $repr` to `*mut $ty`, +/// and this operation must preserve referent size (ie, `size_of_val_raw`). +macro_rules! unsafe_impl_known_layout { + ($($tyvar:ident: ?Sized + KnownLayout =>)? #[repr($repr:ty)] $ty:ty) => {{ + use core::ptr::NonNull; + + crate::util::macros::__unsafe(); + + #[allow(non_local_definitions)] + // SAFETY: The caller promises that this is sound. + unsafe impl<$($tyvar: ?Sized + KnownLayout)?> KnownLayout for $ty { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + + type PointerMetadata = <$repr as KnownLayout>::PointerMetadata; + type MaybeUninit = <$repr as KnownLayout>::MaybeUninit; + + const LAYOUT: DstLayout = <$repr as KnownLayout>::LAYOUT; + + // SAFETY: All operations preserve address and provenance. Caller + // has promised that the `as` cast preserves size. + // + // FIXME(#429): Add documentation to `NonNull::new_unchecked` that + // it preserves provenance. + #[inline(always)] + fn raw_from_ptr_len(bytes: NonNull, meta: <$repr as KnownLayout>::PointerMetadata) -> NonNull { + #[allow(clippy::as_conversions)] + let ptr = <$repr>::raw_from_ptr_len(bytes, meta).as_ptr() as *mut Self; + // SAFETY: `ptr` was converted from `bytes`, which is non-null. + unsafe { NonNull::new_unchecked(ptr) } + } + + #[inline(always)] + fn pointer_to_metadata(ptr: *mut Self) -> Self::PointerMetadata { + #[allow(clippy::as_conversions)] + let ptr = ptr as *mut $repr; + <$repr>::pointer_to_metadata(ptr) + } + } + }}; +} + +/// Uses `align_of` to confirm that a type or set of types have alignment 1. +/// +/// Note that `align_of` requires `T: Sized`, so this macro doesn't work for +/// unsized types. +macro_rules! assert_unaligned { + ($($tys:ty),*) => { + $( + // We only compile this assertion under `cfg(test)` to avoid taking + // an extra non-dev dependency (and making this crate more expensive + // to compile for our dependents). + #[cfg(test)] + static_assertions::const_assert_eq!(core::mem::align_of::<$tys>(), 1); + )* + }; +} + +/// Emits a function definition as either `const fn` or `fn` depending on +/// whether the current toolchain version supports `const fn` with generic trait +/// bounds. +macro_rules! maybe_const_trait_bounded_fn { + // This case handles both `self` methods (where `self` is by value) and + // non-method functions. Each `$args` may optionally be followed by `: + // $arg_tys:ty`, which can be omitted for `self`. + ($(#[$attr:meta])* $vis:vis const fn $name:ident($($args:ident $(: $arg_tys:ty)?),* $(,)?) $(-> $ret_ty:ty)? $body:block) => { + #[cfg(not(no_zerocopy_generic_bounds_in_const_fn_1_61_0))] + $(#[$attr])* $vis const fn $name($($args $(: $arg_tys)?),*) $(-> $ret_ty)? $body + + #[cfg(no_zerocopy_generic_bounds_in_const_fn_1_61_0)] + $(#[$attr])* $vis fn $name($($args $(: $arg_tys)?),*) $(-> $ret_ty)? $body + }; +} + +/// Either panic (if the current Rust toolchain supports panicking in `const +/// fn`) or evaluate a constant that will cause an array indexing error whose +/// error message will include the format string. +/// +/// The type that this expression evaluates to must be `Copy`, or else the +/// non-panicking desugaring will fail to compile. +macro_rules! const_panic { + (@non_panic $($_arg:tt)+) => {{ + // This will type check to whatever type is expected based on the call + // site. + let panic: [_; 0] = []; + // This will always fail (since we're indexing into an array of size 0. + #[allow(unconditional_panic)] + panic[0] + }}; + ($($arg:tt)+) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + panic!($($arg)+); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + const_panic!(@non_panic $($arg)+) + }}; +} + +/// Either assert (if the current Rust toolchain supports panicking in `const +/// fn`) or evaluate the expression and, if it evaluates to `false`, call +/// `const_panic!`. This is used in place of `assert!` in const contexts to +/// accommodate old toolchains. +macro_rules! const_assert { + ($e:expr) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + assert!($e); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e))); + } + } + }}; + ($e:expr, $($args:tt)+) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + assert!($e, $($args)+); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e), ": ", stringify!($arg)), $($args)*); + } + } + }}; +} + +/// Like `const_assert!`, but relative to `debug_assert!`. +macro_rules! const_debug_assert { + ($e:expr $(, $msg:expr)?) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + debug_assert!($e $(, $msg)?); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + // Use this (rather than `#[cfg(debug_assertions)]`) to ensure that + // `$e` is always compiled even if it will never be evaluated at + // runtime. + if cfg!(debug_assertions) { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e) $(, ": ", $msg)?)); + } + } + } + }} +} + +/// Either invoke `unreachable!()` or `loop {}` depending on whether the Rust +/// toolchain supports panicking in `const fn`. +macro_rules! const_unreachable { + () => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + unreachable!(); + + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + loop {} + }}; +} + +/// Asserts at compile time that `$condition` is true for `Self` or the given +/// `$tyvar`s. Unlike `const_assert`, this is *strictly* a compile-time check; +/// it cannot be evaluated in a runtime context. The condition is checked after +/// monomorphization and, upon failure, emits a compile error. +macro_rules! static_assert { + (Self $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )? => $condition:expr $(, $args:tt)*) => {{ + trait StaticAssert { + const ASSERT: bool; + } + + impl StaticAssert for T { + const ASSERT: bool = { + const_assert!($condition $(, $args)*); + $condition + }; + } + + const_assert!(::ASSERT); + }}; + ($($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* => $condition:expr $(, $args:tt)*) => {{ + trait StaticAssert { + const ASSERT: bool; + } + + // NOTE: We use `PhantomData` so we can support unsized types. + impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?,)*> StaticAssert for ($(core::marker::PhantomData<$tyvar>,)*) { + const ASSERT: bool = { + const_assert!($condition $(, $args)*); + $condition + }; + } + + const_assert!(<($(core::marker::PhantomData<$tyvar>,)*) as StaticAssert>::ASSERT); + }}; +} + +/// Assert at compile time that `tyvar` does not have a zero-sized DST +/// component. +macro_rules! static_assert_dst_is_not_zst { + ($tyvar:ident) => {{ + use crate::KnownLayout; + static_assert!($tyvar: ?Sized + KnownLayout => { + let dst_is_zst = match $tyvar::LAYOUT.size_info { + crate::SizeInfo::Sized { .. } => false, + crate::SizeInfo::SliceDst(TrailingSliceLayout { elem_size, .. }) => { + elem_size == 0 + } + }; + !dst_is_zst + }, "cannot call this method on a dynamically-sized type whose trailing slice element is zero-sized"); + }} +} + +/// Defines a named [`Cast`] implementation. +/// +/// # Safety +/// +/// The caller must ensure that, given `src: *mut $src`, `src as *mut $dst` is a +/// size-preserving or size-shrinking cast. +/// +/// [`Cast`]: crate::pointer::cast::Cast +#[macro_export] +#[doc(hidden)] +macro_rules! define_cast { + // We require the caller to provide an `unsafe` block as part of the input + // syntax since a call to `define_cast!` is useless inside of an `unsafe` + // block (since it would introduce a type which can't be named outside of + // the context of that block). + (unsafe { $vis:vis $name:ident $(<$tyvar:ident $(: ?$optbound:ident)?>)? = $src:ty => $dst:ty }) => { + #[allow(missing_debug_implementations, missing_copy_implementations, unreachable_pub)] + $vis enum $name {} + + // SAFETY: The caller promises that `src as *mut $src` is a size- + // preserving or size-shrinking cast. All operations preserve + // provenance. + unsafe impl $(<$tyvar $(: ?$optbound)?>)? $crate::pointer::cast::Project<$src, $dst> for $name { + fn project(src: $crate::pointer::PtrInner<'_, $src>) -> *mut $dst { + #[allow(clippy::as_conversions)] + return src.as_ptr() as *mut $dst; + } + } + + // SAFETY: The impl of `Project::project` preserves referent address. + unsafe impl $(<$tyvar $(: ?$optbound)?>)? $crate::pointer::cast::Cast<$src, $dst> for $name {} + }; +} + +/// Implements `TransmuteFrom` and `SizeEq` for `T` and `$wrapper`. +/// +/// # Safety +/// +/// `T` and `$wrapper` must have the same bit validity, and must have the +/// same size in the sense of `CastExact` (specifically, both a +/// `T`-to-`$wrapper` cast and a `$wrapper`-to-`T` cast must be +/// size-preserving). +macro_rules! unsafe_impl_for_transparent_wrapper { + ($vis:vis T $(: ?$optbound:ident)? => $wrapper:ident) => {{ + crate::util::macros::__unsafe(); + + use crate::pointer::{TransmuteFrom, cast::{CastExact, TransitiveProject}, SizeEq, invariant::Valid}; + use crate::wrappers::ReadOnly; + + // SAFETY: The caller promises that `T` and `$wrapper` have the same + // bit validity. + unsafe impl TransmuteFrom for $wrapper {} + // SAFETY: See previous safety comment. + unsafe impl TransmuteFrom<$wrapper, Valid, Valid> for T {} + // SAFETY: The caller promises that a `T` to `$wrapper` cast is + // size-preserving. + define_cast!(unsafe { $vis CastToWrapper = T => $wrapper }); + // SAFETY: The caller promises that a `T` to `$wrapper` cast is + // size-preserving. + unsafe impl CastExact> for CastToWrapper {} + // SAFETY: The caller promises that a `$wrapper` to `T` cast is + // size-preserving. + define_cast!(unsafe { $vis CastFromWrapper = $wrapper => T }); + // SAFETY: The caller promises that a `$wrapper` to `T` cast is + // size-preserving. + unsafe impl CastExact<$wrapper, T> for CastFromWrapper {} + + impl SizeEq for $wrapper { + type CastFrom = CastToWrapper; + } + impl SizeEq<$wrapper> for T { + type CastFrom = CastFromWrapper; + } + + impl SizeEq> for $wrapper { + type CastFrom = TransitiveProject< + T, + >>::CastFrom, + CastToWrapper, + >; + } + impl SizeEq<$wrapper> for ReadOnly { + type CastFrom = TransitiveProject< + T, + CastFromWrapper, + as SizeEq>::CastFrom, + >; + } + + impl SizeEq> for ReadOnly<$wrapper> { + type CastFrom = TransitiveProject< + $wrapper, + <$wrapper as SizeEq>>::CastFrom, + > as SizeEq<$wrapper>>::CastFrom, + >; + } + impl SizeEq>> for ReadOnly { + type CastFrom = TransitiveProject< + $wrapper, + <$wrapper as SizeEq>>>::CastFrom, + as SizeEq<$wrapper>>::CastFrom, + >; + } + }}; +} + +macro_rules! impl_transitive_transmute_from { + ($($tyvar:ident $(: ?$optbound:ident)?)? => $t:ty => $u:ty => $v:ty) => { + const _: () = { + use crate::pointer::{TransmuteFrom, SizeEq, invariant::Valid}; + + impl<$($tyvar $(: ?$optbound)?)?> SizeEq<$t> for $v + where + $u: SizeEq<$t>, + $v: SizeEq<$u>, + { + type CastFrom = cast::TransitiveProject< + $u, + <$u as SizeEq<$t>>::CastFrom, + <$v as SizeEq<$u>>::CastFrom + >; + } + + // SAFETY: Since `$u: TransmuteFrom<$t, Valid, Valid>`, it is sound + // to transmute a bit-valid `$t` to a bit-valid `$u`. Since `$v: + // TransmuteFrom<$u, Valid, Valid>`, it is sound to transmute that + // bit-valid `$u` to a bit-valid `$v`. + unsafe impl<$($tyvar $(: ?$optbound)?)?> TransmuteFrom<$t, Valid, Valid> for $v + where + $u: TransmuteFrom<$t, Valid, Valid>, + $v: TransmuteFrom<$u, Valid, Valid>, + {} + }; + }; +} + +/// A no-op `unsafe fn` for use in macro expansions. +/// +/// Calling this function in a macro expansion ensures that the macro's caller +/// must wrap the call in `unsafe { ... }`. +#[inline(always)] +pub(crate) const unsafe fn __unsafe() {} + +/// Extracts the contents of doc comments. +#[allow(unused)] +macro_rules! docstring { + ($(#[doc = $content:expr])*) => { + concat!($($content, "\n",)*) + } +} + +/// Generate a rustdoc-style header with `$name` as the HTML ID for the 'Code +/// Generation' section of documentation. +#[allow(unused)] +macro_rules! codegen_header { + ($level:expr, $name:expr) => { + concat!( + " +<", + $level, + " id='method.", + $name, + ".codegen'> + § + Code Generation + +" + ) + }; +} + +/// Generates HTML tabs. +#[rustfmt::skip] +#[allow(unused)] +macro_rules! tabs { + ( + name = $name:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $n:literal + @title $title:literal + $(#[doc = $content:expr])* + ]),* + ) => { + concat!(" +

", $(concat!(" +
+
", $title, "
 +
+ +", $($content, "\n",)* " +\ +
+
"),)* +"
") + } +} + +/// Generates the HTML for a single benchmark example. +#[allow(unused)] +macro_rules! codegen_example { + (format = $format:expr, bench = $bench:expr) => { + tabs!( + name = $bench, + arity = 4, + [ + @index 1 + @title "Format" + /// ```ignore + #[doc = include_str!(concat!("../benches/formats/", $format, ".rs"))] + /// ``` + ], + [ + @index 2 + @title "Benchmark" + /// ```ignore + #[doc = include_str!(concat!("../benches/", $bench, ".rs"))] + /// ``` + ], + [ + open + @index 3 + @title "Assembly" + /// ```plain + #[doc = include_str!(concat!("../benches/", $bench, ".x86-64"))] + /// ``` + ], + [ + @index 4 + @title "Machine Code Analysis" + /// ```plain + #[doc = include_str!(concat!("../benches/", $bench, ".x86-64.mca"))] + /// ``` + ] + ) + } +} + +/// Generate the HTML for a suite of benchmark examples. +#[allow(unused)] +macro_rules! codegen_example_suite { + ( + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + tabs!( + name = $bench, + arity = $arity, + $([ + $($open)* + @index $index + @title $title + #[doc = codegen_example!( + format = concat!($format, "_", $variant), + bench = concat!($bench, "_", $variant) + )] + ]),* + ) + } +} + +/// Generates the string for code generation preamble. +#[allow(unused)] +macro_rules! codegen_preamble { + () => { + docstring!( + /// + /// This abstraction is safe and cheap, but does not necessarily + /// have zero runtime cost. The codegen you experience in practice + /// will depend on optimization level, the layout of the destination + /// type, and what the compiler can prove about the source. + /// + ) + } +} + +/// Stub for rendering codegen documentation; used to break build dependency +/// between benches and zerocopy when re-blessing codegen tests. +#[allow(unused)] +#[cfg(not(doc))] +macro_rules! codegen_section { + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + "" + }; + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + ) => { + "" + }; +} + +/// Generates the HTML for code generation documentation. +#[allow(unused)] +#[cfg(doc)] +macro_rules! codegen_section { + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + concat!( + codegen_header!($level, $bench), + codegen_preamble!(), + docstring!( + /// + /// The below examples illustrate typical codegen for + /// increasingly complex types: + /// + ), + codegen_example_suite!( + bench = $bench, + format = $format, + arity = $arity, + $([ + $($open)* + @index $index + @title $title + @variant $variant + ]),* + ) + ) + }; + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + ) => { + concat!( + codegen_header!($level, $bench), + codegen_preamble!(), + codegen_example!( + format = $format, + bench = $bench + ) + ) + } +} diff --git a/rust/zerocopy/src/util/mod.rs b/rust/zerocopy/src/util/mod.rs new file mode 100644 index 000000000000..1a6c0b22a47b --- /dev/null +++ b/rust/zerocopy/src/util/mod.rs @@ -0,0 +1,936 @@ +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#[macro_use] +pub(crate) mod macros; + +#[doc(hidden)] +pub mod macro_util; + +use core::{ + marker::PhantomData, + mem::{self, ManuallyDrop}, + num::NonZeroUsize, + ptr::NonNull, +}; + +use super::*; +use crate::pointer::{ + invariant::{Exclusive, Shared, Valid}, + SizeEq, TransmuteFromPtr, +}; + +/// Like [`PhantomData`], but [`Send`] and [`Sync`] regardless of whether the +/// wrapped `T` is. +pub(crate) struct SendSyncPhantomData(PhantomData); + +// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound +// to be called from multiple threads. +unsafe impl Send for SendSyncPhantomData {} +// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound +// to be called from multiple threads. +unsafe impl Sync for SendSyncPhantomData {} + +impl Default for SendSyncPhantomData { + fn default() -> SendSyncPhantomData { + SendSyncPhantomData(PhantomData) + } +} + +impl PartialEq for SendSyncPhantomData { + fn eq(&self, _other: &Self) -> bool { + true + } +} + +impl Eq for SendSyncPhantomData {} + +impl Clone for SendSyncPhantomData { + fn clone(&self) -> Self { + SendSyncPhantomData(PhantomData) + } +} + +#[cfg(miri)] +extern "Rust" { + /// Miri-provided intrinsic that marks the pointer `ptr` as aligned to + /// `align`. + /// + /// This intrinsic is used to inform Miri's symbolic alignment checker that + /// a pointer is aligned, even if Miri cannot statically deduce that fact. + /// This is often required when performing raw pointer arithmetic or casts + /// where the alignment is guaranteed by runtime checks or invariants that + /// Miri is not aware of. + pub(crate) fn miri_promise_symbolic_alignment(ptr: *const (), align: usize); +} + +pub(crate) trait AsAddress { + fn addr(self) -> usize; +} + +impl AsAddress for &T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +impl AsAddress for &mut T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +impl AsAddress for NonNull { + #[inline(always)] + fn addr(self) -> usize { + AsAddress::addr(self.as_ptr()) + } +} + +impl AsAddress for *const T { + #[inline(always)] + fn addr(self) -> usize { + // FIXME(#181), FIXME(https://github.com/rust-lang/rust/issues/95228): + // Use `.addr()` instead of `as usize` once it's stable, and get rid of + // this `allow`. Currently, `as usize` is the only way to accomplish + // this. + #[allow(clippy::as_conversions)] + #[cfg_attr( + __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, + allow(lossy_provenance_casts) + )] + return self.cast::<()>() as usize; + } +} + +impl AsAddress for *mut T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +/// Validates that `t` is aligned to `align_of::()`. +#[inline(always)] +pub(crate) fn validate_aligned_to(t: T) -> Result<(), AlignmentError<(), U>> { + // `mem::align_of::()` is guaranteed to return a non-zero value, which in + // turn guarantees that this mod operation will not panic. + #[allow(clippy::arithmetic_side_effects)] + let remainder = t.addr() % mem::align_of::(); + if remainder == 0 { + Ok(()) + } else { + // SAFETY: We just confirmed that `t.addr() % align_of::() != 0`. + // That's only possible if `align_of::() > 1`. + Err(unsafe { AlignmentError::new_unchecked(()) }) + } +} + +/// Returns the bytes needed to pad `len` to the next multiple of `align`. +/// +/// This function assumes that align is a power of two; there are no guarantees +/// on the answer it gives if this is not the case. +#[cfg_attr( + kani, + kani::requires(len <= DstLayout::MAX_SIZE), + kani::requires(align.is_power_of_two()), + kani::ensures(|&p| (len + p) % align.get() == 0), + // Ensures that we add the minimum required padding. + kani::ensures(|&p| p < align.get()), +)] +pub(crate) const fn padding_needed_for(len: usize, align: NonZeroUsize) -> usize { + #[cfg(kani)] + #[kani::proof_for_contract(padding_needed_for)] + fn proof() { + padding_needed_for(kani::any(), kani::any()); + } + + // Abstractly, we want to compute: + // align - (len % align). + // Handling the case where len%align is 0. + // Because align is a power of two, len % align = len & (align-1). + // Guaranteed not to underflow as align is nonzero. + #[allow(clippy::arithmetic_side_effects)] + let mask = align.get() - 1; + + // To efficiently subtract this value from align, we can use the bitwise + // complement. + // Note that ((!len) & (align-1)) gives us a number that with (len & + // (align-1)) sums to align-1. So subtracting 1 from x before taking the + // complement subtracts `len` from `align`. Some quick inspection of + // cases shows that this also handles the case where `len % align = 0` + // correctly too: len-1 % align then equals align-1, so the complement mod + // align will be 0, as desired. + // + // The following reasoning can be verified quickly by an SMT solver + // supporting the theory of bitvectors: + // ```smtlib + // ; Naive implementation of padding + // (define-fun padding1 ( + // (len (_ BitVec 32)) + // (align (_ BitVec 32))) (_ BitVec 32) + // (ite + // (= (_ bv0 32) (bvand len (bvsub align (_ bv1 32)))) + // (_ bv0 32) + // (bvsub align (bvand len (bvsub align (_ bv1 32)))))) + // + // ; The implementation below + // (define-fun padding2 ( + // (len (_ BitVec 32)) + // (align (_ BitVec 32))) (_ BitVec 32) + // (bvand (bvnot (bvsub len (_ bv1 32))) (bvsub align (_ bv1 32)))) + // + // (define-fun is-power-of-two ((x (_ BitVec 32))) Bool + // (= (_ bv0 32) (bvand x (bvsub x (_ bv1 32))))) + // + // (declare-const len (_ BitVec 32)) + // (declare-const align (_ BitVec 32)) + // ; Search for a case where align is a power of two and padding2 disagrees + // ; with padding1 + // (assert (and (is-power-of-two align) + // (not (= (padding1 len align) (padding2 len align))))) + // (simplify (padding1 (_ bv300 32) (_ bv32 32))) ; 20 + // (simplify (padding2 (_ bv300 32) (_ bv32 32))) ; 20 + // (simplify (padding1 (_ bv322 32) (_ bv32 32))) ; 30 + // (simplify (padding2 (_ bv322 32) (_ bv32 32))) ; 30 + // (simplify (padding1 (_ bv8 32) (_ bv8 32))) ; 0 + // (simplify (padding2 (_ bv8 32) (_ bv8 32))) ; 0 + // (check-sat) ; unsat, also works for 64-bit bitvectors + // ``` + !(len.wrapping_sub(1)) & mask +} + +/// Rounds `n` down to the largest value `m` such that `m <= n` and `m % align +/// == 0`. +/// +/// # Panics +/// +/// May panic if `align` is not a power of two. Even if it doesn't panic in this +/// case, it will produce nonsense results. +#[inline(always)] +#[cfg_attr( + kani, + kani::requires(align.is_power_of_two()), + kani::ensures(|&m| m <= n && m % align.get() == 0), + // Guarantees that `m` is the *largest* value such that `m % align == 0`. + kani::ensures(|&m| { + // If this `checked_add` fails, then the next multiple would wrap + // around, which trivially satisfies the "largest value" requirement. + m.checked_add(align.get()).map(|next_mul| next_mul > n).unwrap_or(true) + }) +)] +pub(crate) const fn round_down_to_next_multiple_of_alignment( + n: usize, + align: NonZeroUsize, +) -> usize { + #[cfg(kani)] + #[kani::proof_for_contract(round_down_to_next_multiple_of_alignment)] + fn proof() { + round_down_to_next_multiple_of_alignment(kani::any(), kani::any()); + } + + let align = align.get(); + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + debug_assert!(align.is_power_of_two()); + + // Subtraction can't underflow because `align.get() >= 1`. + #[allow(clippy::arithmetic_side_effects)] + let mask = !(align - 1); + n & mask +} + +pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { + if a.get() < b.get() { + b + } else { + a + } +} + +pub(crate) const fn min(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { + if a.get() > b.get() { + b + } else { + a + } +} + +/// Copies `src` into the prefix of `dst`. +/// +/// # Safety +/// +/// The caller guarantees that `src.len() <= dst.len()`. +#[inline(always)] +pub(crate) unsafe fn copy_unchecked(src: &[u8], dst: &mut [u8]) { + debug_assert!(src.len() <= dst.len()); + // SAFETY: This invocation satisfies the safety contract of + // copy_nonoverlapping [1]: + // - `src.as_ptr()` is trivially valid for reads of `src.len()` bytes + // - `dst.as_ptr()` is valid for writes of `src.len()` bytes, because the + // caller has promised that `src.len() <= dst.len()` + // - `src` and `dst` are, trivially, properly aligned + // - the region of memory beginning at `src` with a size of `src.len()` + // bytes does not overlap with the region of memory beginning at `dst` + // with the same size, because `dst` is derived from an exclusive + // reference. + unsafe { + core::ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), src.len()); + }; +} + +/// Unsafely transmutes the given `src` into a type `Dst`. +/// +/// # Safety +/// +/// The value `src` must be a valid instance of `Dst`. +#[inline(always)] +pub(crate) const unsafe fn transmute_unchecked(src: Src) -> Dst { + static_assert!(Src, Dst => core::mem::size_of::() == core::mem::size_of::()); + + #[repr(C)] + union Transmute { + src: ManuallyDrop, + dst: ManuallyDrop, + } + + // SAFETY: Since `Transmute` is `#[repr(C)]`, its `src` and `dst` + // fields both start at the same offset and the types of those fields are + // transparent wrappers around `Src` and `Dst` [1]. Consequently, + // initializing `Transmute` with with `src` and then reading out `dst` is + // equivalent to transmuting from `Src` to `Dst` [2]. Transmuting from `src` + // to `Dst` is valid because — by contract on the caller — `src` is a valid + // instance of `Dst`. + // + // [1] Per https://doc.rust-lang.org/1.82.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop` is guaranteed to have the same layout and bit + // validity as `T`, and is subject to the same layout optimizations as + // `T`. + // + // [2] Per https://doc.rust-lang.org/1.82.0/reference/items/unions.html#reading-and-writing-union-fields: + // + // Effectively, writing to and then reading from a union with the C + // representation is analogous to a transmute from the type used for + // writing to the type used for reading. + unsafe { ManuallyDrop::into_inner(Transmute { src: ManuallyDrop::new(src) }.dst) } +} + +/// # Safety +/// +/// `Src` must have a greater or equal alignment to `Dst`. +pub(crate) unsafe fn transmute_ref(src: &Src) -> &Dst +where + Src: ?Sized, + Dst: SizeEq + + TransmuteFromPtr>::CastFrom, R> + + ?Sized, +{ + let dst = Ptr::from_ref(src).transmute(); + // SAFETY: The caller promises that `Src`'s alignment is at least as large + // as `Dst`'s alignment. + let dst = unsafe { dst.assume_alignment() }; + dst.as_ref() +} + +/// # Safety +/// +/// `Src` must have a greater or equal alignment to `Dst`. +pub(crate) unsafe fn transmute_mut(src: &mut Src) -> &mut Dst +where + Src: ?Sized, + Dst: SizeEq + + TransmuteFromPtr>::CastFrom, R> + + ?Sized, +{ + let dst = Ptr::from_mut(src).transmute(); + // SAFETY: The caller promises that `Src`'s alignment is at least as large + // as `Dst`'s alignment. + let dst = unsafe { dst.assume_alignment() }; + dst.as_mut() +} + +/// Uses `allocate` to create a `Box`. +/// +/// # Errors +/// +/// Returns an error on allocation failure. Allocation failure is guaranteed +/// never to cause a panic or an abort. +/// +/// # Safety +/// +/// `allocate` must be either `alloc::alloc::alloc` or +/// `alloc::alloc::alloc_zeroed`. The referent of the box returned by `new_box` +/// has the same bit-validity as the referent of the pointer returned by the +/// given `allocate` and sufficient size to store `T` with `meta`. +#[must_use = "has no side effects (other than allocation)"] +#[cfg(feature = "alloc")] +#[inline] +pub(crate) unsafe fn new_box( + meta: T::PointerMetadata, + allocate: unsafe fn(core::alloc::Layout) -> *mut u8, +) -> Result, AllocError> +where + T: ?Sized + crate::KnownLayout, +{ + let align = T::LAYOUT.align.get(); + if !T::is_valid_metadata(meta) { + return Err(AllocError); + } + let size = match T::size_for_metadata(meta) { + Some(size) => size, + // Thanks to the `!T::is_valid_metadata(meta)` check + // above, this branch is unreachable. Fortunately, the + // optimizer recognizes this, so replacing this branch + // with `unreachable_unchecked` produces no codegen + // improvements. + None => return Err(AllocError), + }; + let ptr = if size != 0 { + // SAFETY: + // - `align` is derived from a `NonZeroUsize` and is thus non-zero. + // - `align` is a power of two because, by invariant on + // `KnownLayout::LAYOUT` `::LAYOUT` accurately + // reflects the layout of `T`. + // - `size`, by invariant on `size_for_metadata` is well-aligned for + // `align` and, by the check on `T::is_valid_metadata(meta)`, is less + // than `isize::MAX`. + let layout: Layout = unsafe { Layout::from_size_align_unchecked(size, align) }; + // SAFETY: By contract on the caller, `allocate` is either + // `alloc::alloc::alloc` or `alloc::alloc::alloc_zeroed`. The above + // check ensures their shared safety precondition: that the supplied + // layout is not zero-sized type [1]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/alloc/trait.GlobalAlloc.html#tymethod.alloc: + // + // This function is unsafe because undefined behavior can result if + // the caller does not ensure that layout has non-zero size. + let ptr = unsafe { allocate(layout) }; + match NonNull::new(ptr) { + Some(ptr) => ptr, + None => return Err(AllocError), + } + } else { + // We use `transmute` instead of an `as` cast since Miri (with strict + // provenance enabled) notices and complains that an `as` cast creates a + // pointer with no provenance. Miri isn't smart enough to realize that + // we're only executing this branch when we're constructing a zero-sized + // `Box`, which doesn't require provenance. + // + // SAFETY: any initialized bit sequence is a bit-valid `*mut u8`. All + // bits of a `usize` are initialized. + // + // `#[allow(unknown_lints)]` is for `integer_to_ptr_transmutes` + #[allow(unknown_lints)] + #[allow(clippy::useless_transmute, integer_to_ptr_transmutes)] + let dangling = unsafe { mem::transmute::(align) }; + // SAFETY: `dangling` is constructed from `align`, which is derived from + // a `NonZeroUsize`, which is guaranteed to be non-zero. + // + // `Box<[T]>` does not allocate when `T` is zero-sized or when `len` is + // zero, but it does require a non-null dangling pointer for its + // allocation. + // + // FIXME(https://github.com/rust-lang/rust/issues/95228): Use + // `std::ptr::without_provenance` once it's stable. That may optimize + // better. As written, Rust may assume that this consumes "exposed" + // provenance, and thus Rust may have to assume that this may consume + // provenance from any pointer whose provenance has been exposed. + unsafe { NonNull::new_unchecked(dangling) } + }; + + let ptr = T::raw_from_ptr_len(ptr, meta); + + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. Make sure to + // include a justification that `ptr.as_ptr()` is validly-aligned in the ZST + // case (in which we manually construct a dangling pointer) and to justify + // why `Box` is safe to drop (it's because `allocate` uses the system + // allocator). + #[allow(clippy::undocumented_unsafe_blocks)] + Ok(unsafe { alloc::boxed::Box::from_raw(ptr.as_ptr()) }) +} + +mod len_of { + use super::*; + + /// A witness type for metadata of a valid instance of `&T`. + pub struct MetadataOf { + /// # Safety + /// + /// The size of an instance of `&T` with the given metadata is not + /// larger than `isize::MAX`. + meta: T::PointerMetadata, + _p: PhantomData, + } + + impl Copy for MetadataOf {} + impl Clone for MetadataOf { + #[inline] + fn clone(&self) -> Self { + *self + } + } + + impl core::fmt::Debug for MetadataOf + where + T::PointerMetadata: core::fmt::Debug, + { + #[inline] + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetadataOf").field("meta", &self.meta).finish() + } + } + + impl MetadataOf + where + T: KnownLayout, + { + /// Returns `None` if `meta` is greater than `t`'s metadata. + #[inline(always)] + pub(crate) fn new_in_bounds(t: &T, meta: usize) -> Option + where + T: KnownLayout, + { + if meta <= Ptr::from_ref(t).len() { + // SAFETY: We have checked that `meta` is not greater than `t`'s + // metadata, which, by invariant on `&T`, addresses no more than + // `isize::MAX` bytes [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: + // &mut T`, when such values cross an API boundary, the + // following invariants must generally be upheld: + // + // * `t` is non-null + // * `t` is aligned to `align_of_val(t)` + // * if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being "dereferenceable" for + // N bytes means that the memory range `[a, a + N)` is all + // contained within a single allocated object. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/ptr/index.html#allocated-object: + // + // For any allocated object with `base` address, `size`, and + // a set of `addresses`, the following are guaranteed: + // - For all addresses `a` in `addresses`, `a` is in the + // range `base .. (base + size)` (note that this requires + // `a < base + size`, not `a <= base + size`) + // - `base` is not equal to [`null()`] (i.e., the address + // with the numerical value 0) + // - `base + size <= usize::MAX` + // - `size <= isize::MAX` + Some(unsafe { Self::new_unchecked(meta) }) + } else { + None + } + } + + /// # Safety + /// + /// The size of an instance of `&T` with the given metadata is not + /// larger than `isize::MAX`. + pub(crate) unsafe fn new_unchecked(meta: T::PointerMetadata) -> Self { + // SAFETY: The caller has promised that the size of an instance of + // `&T` with the given metadata is not larger than `isize::MAX`. + Self { meta, _p: PhantomData } + } + + pub(crate) fn get(&self) -> T::PointerMetadata + where + T::PointerMetadata: Copy, + { + self.meta + } + + #[inline] + pub(crate) fn padding_needed_for(&self) -> usize + where + T: KnownLayout, + { + let trailing_slice_layout = crate::trailing_slice_layout::(); + + // FIXME(#67): Remove this allow. See NumExt for more details. + #[allow( + unstable_name_collisions, + clippy::incompatible_msrv, + clippy::multiple_unsafe_ops_per_block + )] + // SAFETY: By invariant on `self`, a `&T` with metadata `self.meta` + // describes an object of size `<= isize::MAX`. This computes the + // size of such a `&T` without any trailing padding, and so neither + // the multiplication nor the addition will overflow. + let unpadded_size = unsafe { + let trailing_size = self.meta.unchecked_mul(trailing_slice_layout.elem_size); + trailing_size.unchecked_add(trailing_slice_layout.offset) + }; + + util::padding_needed_for(unpadded_size, T::LAYOUT.align) + } + + #[inline(always)] + pub(crate) fn validate_cast_and_convert_metadata( + addr: usize, + bytes_len: MetadataOf<[u8]>, + cast_type: CastType, + meta: Option, + ) -> Result<(MetadataOf, MetadataOf<[u8]>), MetadataCastError> { + let layout = match meta { + None => T::LAYOUT, + // This can return `Err(MetadataCastError::Size)` if the + // metadata describes an object which can't fit in an `isize`. + Some(meta) => { + if !T::is_valid_metadata(meta) { + return Err(MetadataCastError::Size); + } + let size = match T::size_for_metadata(meta) { + Some(size) => size, + // Thanks to the `!T::is_valid_metadata(meta)` check + // above, this branch is unreachable. Fortunately, the + // optimizer recognizes this, so replacing this branch + // with `unreachable_unchecked` produces no codegen + // improvements. + None => return Err(MetadataCastError::Size), + }; + DstLayout { + align: T::LAYOUT.align, + size_info: crate::SizeInfo::Sized { size }, + statically_shallow_unpadded: false, + } + } + }; + // Lemma 0: By contract on `validate_cast_and_convert_metadata`, if + // the result is `Ok(..)`, then a `&T` with `elems` trailing slice + // elements is no larger in size than `bytes_len.get()`. + let (elems, split_at) = + layout.validate_cast_and_convert_metadata(addr, bytes_len.get(), cast_type)?; + let elems = T::PointerMetadata::from_elem_count(elems); + + // For a slice DST type, if `meta` is `Some(elems)`, then we + // synthesize `layout` to describe a sized type whose size is equal + // to the size of the instance that we are asked to cast. For sized + // types, `validate_cast_and_convert_metadata` returns `elems == 0`. + // Thus, in this case, we need to use the `elems` passed by the + // caller, not the one returned by + // `validate_cast_and_convert_metadata`. + // + // Lemma 1: A `&T` with `elems` trailing slice elements is no larger + // in size than `bytes_len.get()`. Proof: + // - If `meta` is `None`, then `elems` satisfies this condition by + // Lemma 0. + // - If `meta` is `Some(meta)`, then `layout` describes an object + // whose size is equal to the size of an `&T` with `meta` + // metadata. By Lemma 0, that size is not larger than + // `bytes_len.get()`. + // + // Lemma 2: A `&T` with `elems` trailing slice elements is no larger + // than `isize::MAX` bytes. Proof: By Lemma 1, a `&T` with metadata + // `elems` is not larger in size than `bytes_len.get()`. By + // invariant on `MetadataOf<[u8]>`, a `&[u8]` with metadata + // `bytes_len` is not larger than `isize::MAX`. Because + // `size_of::()` is `1`, a `&[u8]` with metadata `bytes_len` has + // size `bytes_len.get()` bytes. Therefore, a `&T` with metadata + // `elems` has size not larger than `isize::MAX`. + let elems = meta.unwrap_or(elems); + + // SAFETY: See Lemma 2. + let elems = unsafe { MetadataOf::new_unchecked(elems) }; + + // SAFETY: Let `size` be the size of a `&T` with metadata `elems`. + // By post-condition on `validate_cast_and_convert_metadata`, one of + // the following conditions holds: + // - `split_at == size`, in which case, by Lemma 2, `split_at <= + // isize::MAX`. Since `size_of::() == 1`, a `[u8]` with + // `split_at` elems has size not larger than `isize::MAX`. + // - `split_at == bytes_len - size`. Since `bytes_len: + // MetadataOf`, and since `size` is non-negative, `split_at` + // addresses no more bytes than `bytes_len` does. Since + // `bytes_len: MetadataOf`, `bytes_len` describes a `[u8]` + // which has no more than `isize::MAX` bytes, and thus so does + // `split_at`. + let split_at = unsafe { MetadataOf::<[u8]>::new_unchecked(split_at) }; + Ok((elems, split_at)) + } + } +} + +pub use len_of::MetadataOf; + +/// Since we support multiple versions of Rust, there are often features which +/// have been stabilized in the most recent stable release which do not yet +/// exist (stably) on our MSRV. This module provides polyfills for those +/// features so that we can write more "modern" code, and just remove the +/// polyfill once our MSRV supports the corresponding feature. Without this, +/// we'd have to write worse/more verbose code and leave FIXME comments +/// sprinkled throughout the codebase to update to the new pattern once it's +/// stabilized. +/// +/// Each trait is imported as `_` at the crate root; each polyfill should "just +/// work" at usage sites. +pub(crate) mod polyfills { + use core::ptr::{self, NonNull}; + + // A polyfill for `NonNull::slice_from_raw_parts` that we can use before our + // MSRV is 1.70, when that function was stabilized. + // + // The `#[allow(unused)]` is necessary because, on sufficiently recent + // toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent + // method rather than to this trait, and so this trait is considered unused. + // + // FIXME(#67): Once our MSRV is 1.70, remove this. + #[allow(unused)] + pub(crate) trait NonNullExt { + fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]>; + } + + impl NonNullExt for NonNull { + // NOTE on coverage: this will never be tested in nightly since it's a + // polyfill for a feature which has been stabilized on our nightly + // toolchain. + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]> { + let ptr = ptr::slice_from_raw_parts_mut(data.as_ptr(), len); + // SAFETY: `ptr` is converted from `data`, which is non-null. + unsafe { NonNull::new_unchecked(ptr) } + } + } + + // A polyfill for `Self::unchecked_sub` that we can use until methods like + // `usize::unchecked_sub` is stabilized. + // + // The `#[allow(unused)]` is necessary because, on sufficiently recent + // toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent + // method rather than to this trait, and so this trait is considered unused. + // + // FIXME(#67): Once our MSRV is high enough, remove this. + #[allow(unused)] + pub(crate) trait NumExt { + /// Add without checking for overflow. + /// + /// # Safety + /// + /// The caller promises that the addition will not overflow. + unsafe fn unchecked_add(self, rhs: Self) -> Self; + + /// Subtract without checking for underflow. + /// + /// # Safety + /// + /// The caller promises that the subtraction will not underflow. + unsafe fn unchecked_sub(self, rhs: Self) -> Self; + + /// Multiply without checking for overflow. + /// + /// # Safety + /// + /// The caller promises that the multiplication will not overflow. + unsafe fn unchecked_mul(self, rhs: Self) -> Self; + } + + // NOTE on coverage: these will never be tested in nightly since they're + // polyfills for a feature which has been stabilized on our nightly + // toolchain. + impl NumExt for usize { + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_add(self, rhs: usize) -> usize { + match self.checked_add(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the addition will not + // underflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_sub(self, rhs: usize) -> usize { + match self.checked_sub(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the subtraction will not + // underflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_mul(self, rhs: usize) -> usize { + match self.checked_mul(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the multiplication will + // not overflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + } +} + +#[cfg(test)] +pub(crate) mod testutil { + use crate::*; + + /// A `T` which is aligned to at least `align_of::()`. + #[derive(Default)] + pub(crate) struct Align { + pub(crate) t: T, + _a: [A; 0], + } + + impl Align { + pub(crate) fn set_default(&mut self) { + self.t = T::default(); + } + } + + impl Align { + pub(crate) const fn new(t: T) -> Align { + Align { t, _a: [] } + } + } + + /// A `T` which is guaranteed not to satisfy `align_of::()`. + /// + /// It must be the case that `align_of::() < align_of::()` in order + /// for this type to work properly. + #[repr(C)] + pub(crate) struct ForceUnalign { + // The outer struct is aligned to `A`, and, thanks to `repr(C)`, `t` is + // placed at the minimum offset that guarantees its alignment. If + // `align_of::() < align_of::()`, then that offset will be + // guaranteed *not* to satisfy `align_of::()`. + // + // Note that we need `T: Unaligned` in order to guarantee that there is + // no padding between `_u` and `t`. + _u: u8, + pub(crate) t: T, + _a: [A; 0], + } + + impl ForceUnalign { + pub(crate) fn new(t: T) -> ForceUnalign { + ForceUnalign { _u: 0, t, _a: [] } + } + } + // A `u64` with alignment 8. + // + // Though `u64` has alignment 8 on some platforms, it's not guaranteed. By + // contrast, `AU64` is guaranteed to have alignment 8 on all platforms. + #[derive( + KnownLayout, + Immutable, + FromBytes, + IntoBytes, + Eq, + PartialEq, + Ord, + PartialOrd, + Default, + Debug, + Copy, + Clone, + )] + #[repr(C, align(8))] + pub(crate) struct AU64(pub(crate) u64); + + impl AU64 { + // Converts this `AU64` to bytes using this platform's endianness. + pub(crate) fn to_bytes(self) -> [u8; 8] { + crate::transmute!(self) + } + } + + impl Display for AU64 { + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(&self.0, f) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_round_down_to_next_multiple_of_alignment() { + fn alt_impl(n: usize, align: NonZeroUsize) -> usize { + let mul = n / align.get(); + mul * align.get() + } + + for align in [1, 2, 4, 8, 16] { + for n in 0..256 { + let align = NonZeroUsize::new(align).unwrap(); + let want = alt_impl(n, align); + let got = round_down_to_next_multiple_of_alignment(n, align); + assert_eq!(got, want, "round_down_to_next_multiple_of_alignment({}, {})", n, align); + } + } + } + + #[rustversion::since(1.57.0)] + #[test] + #[should_panic] + fn test_round_down_to_next_multiple_of_alignment_zerocopy_panic_in_const_and_vec_try_reserve() { + round_down_to_next_multiple_of_alignment(0, NonZeroUsize::new(3).unwrap()); + } + #[test] + fn test_send_sync_phantom_data() { + let x = SendSyncPhantomData::::default(); + let y = x.clone(); + assert!(x == y); + assert!(x == SendSyncPhantomData::::default()); + } + + #[test] + #[allow(clippy::as_conversions)] + fn test_as_address() { + let x = 0u8; + let r = &x; + let mut x_mut = 0u8; + let rm = &mut x_mut; + let p = r as *const u8; + let pm = rm as *mut u8; + let nn = NonNull::new(p as *mut u8).unwrap(); + + assert_eq!(AsAddress::addr(r), p as usize); + assert_eq!(AsAddress::addr(rm), pm as usize); + assert_eq!(AsAddress::addr(p), p as usize); + assert_eq!(AsAddress::addr(pm), pm as usize); + assert_eq!(AsAddress::addr(nn), p as usize); + } +} diff --git a/rust/zerocopy/src/wrappers.rs b/rust/zerocopy/src/wrappers.rs new file mode 100644 index 000000000000..f3930eb7ebc0 --- /dev/null +++ b/rust/zerocopy/src/wrappers.rs @@ -0,0 +1,1032 @@ +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{fmt, hash::Hash}; + +use super::*; +use crate::pointer::{invariant::Valid, SizeEq, TransmuteFrom}; + +/// A type with no alignment requirement. +/// +/// An `Unalign` wraps a `T`, removing any alignment requirement. `Unalign` +/// has the same size and bit validity as `T`, but not necessarily the same +/// alignment [or ABI]. This is useful if a type with an alignment requirement +/// needs to be read from a chunk of memory which provides no alignment +/// guarantees. +/// +/// Since `Unalign` has no alignment requirement, the inner `T` may not be +/// properly aligned in memory. There are five ways to access the inner `T`: +/// - by value, using [`get`] or [`into_inner`] +/// - by reference inside of a callback, using [`update`] +/// - fallibly by reference, using [`try_deref`] or [`try_deref_mut`]; these can +/// fail if the `Unalign` does not satisfy `T`'s alignment requirement at +/// runtime +/// - unsafely by reference, using [`deref_unchecked`] or +/// [`deref_mut_unchecked`]; it is the caller's responsibility to ensure that +/// the `Unalign` satisfies `T`'s alignment requirement +/// - (where `T: Unaligned`) infallibly by reference, using [`Deref::deref`] or +/// [`DerefMut::deref_mut`] +/// +/// [or ABI]: https://github.com/google/zerocopy/issues/164 +/// [`get`]: Unalign::get +/// [`into_inner`]: Unalign::into_inner +/// [`update`]: Unalign::update +/// [`try_deref`]: Unalign::try_deref +/// [`try_deref_mut`]: Unalign::try_deref_mut +/// [`deref_unchecked`]: Unalign::deref_unchecked +/// [`deref_mut_unchecked`]: Unalign::deref_mut_unchecked +/// +/// # Example +/// +/// In this example, we need `EthernetFrame` to have no alignment requirement - +/// and thus implement [`Unaligned`]. `EtherType` is `#[repr(u16)]` and so +/// cannot implement `Unaligned`. We use `Unalign` to relax `EtherType`'s +/// alignment requirement so that `EthernetFrame` has no alignment requirement +/// and can implement `Unaligned`. +/// +/// ```rust +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// # #[derive(FromBytes, KnownLayout, Immutable, Unaligned)] #[repr(C)] struct Mac([u8; 6]); +/// +/// # #[derive(PartialEq, Copy, Clone, Debug)] +/// #[derive(TryFromBytes, KnownLayout, Immutable)] +/// #[repr(u16)] +/// enum EtherType { +/// Ipv4 = 0x0800u16.to_be(), +/// Arp = 0x0806u16.to_be(), +/// Ipv6 = 0x86DDu16.to_be(), +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, KnownLayout, Immutable, Unaligned)] +/// #[repr(C)] +/// struct EthernetFrame { +/// src: Mac, +/// dst: Mac, +/// ethertype: Unalign, +/// payload: [u8], +/// } +/// +/// let bytes = &[ +/// # 0, 1, 2, 3, 4, 5, +/// # 6, 7, 8, 9, 10, 11, +/// # /* +/// ... +/// # */ +/// 0x86, 0xDD, // EtherType +/// 0xDE, 0xAD, 0xBE, 0xEF // Payload +/// ][..]; +/// +/// // PANICS: Guaranteed not to panic because `bytes` is of the right +/// // length, has the right contents, and `EthernetFrame` has no +/// // alignment requirement. +/// let packet = EthernetFrame::try_ref_from_bytes(&bytes).unwrap(); +/// +/// assert_eq!(packet.ethertype.get(), EtherType::Ipv6); +/// assert_eq!(packet.payload, [0xDE, 0xAD, 0xBE, 0xEF]); +/// ``` +/// +/// # Safety +/// +/// `Unalign` is guaranteed to have the same size and bit validity as `T`, +/// and to have [`UnsafeCell`]s covering the same byte ranges as `T`. +/// `Unalign` is guaranteed to have alignment 1. +// NOTE: This type is sound to use with types that need to be dropped. The +// reason is that the compiler-generated drop code automatically moves all +// values to aligned memory slots before dropping them in-place. This is not +// well-documented, but it's hinted at in places like [1] and [2]. However, this +// also means that `T` must be `Sized`; unless something changes, we can never +// support unsized `T`. [3] +// +// [1] https://github.com/rust-lang/rust/issues/54148#issuecomment-420529646 +// [2] https://github.com/google/zerocopy/pull/126#discussion_r1018512323 +// [3] https://github.com/google/zerocopy/issues/209 +#[allow(missing_debug_implementations)] +#[derive(Default, Copy)] +#[cfg_attr(any(feature = "derive", test), derive(Immutable, FromBytes, IntoBytes, Unaligned))] +#[repr(C, packed)] +pub struct Unalign(T); + +// We do not use `derive(KnownLayout)` on `Unalign`, because the derive is not +// smart enough to realize that `Unalign` is always sized and thus emits a +// `KnownLayout` impl bounded on `T: KnownLayout.` This is overly restrictive. +impl_known_layout!(T => Unalign); + +// FIXME(https://github.com/rust-lang/rust-clippy/issues/16087): Move these +// attributes below the comment once this Clippy bug is fixed. +#[cfg_attr( + all(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, any(feature = "derive", test)), + expect(unused_unsafe) +)] +#[cfg_attr( + all( + not(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + any(feature = "derive", test) + ), + allow(unused_unsafe) +)] +// SAFETY: +// - `Unalign` promises to have alignment 1, and so we don't require that `T: +// Unaligned`. +// - `Unalign` has the same bit validity as `T`, and so it is `FromZeros`, +// `FromBytes`, or `IntoBytes` exactly when `T` is as well. +// - `Immutable`: `Unalign` has the same fields as `T`, so it permits +// interior mutation exactly when `T` does. +// - `TryFromBytes`: `Unalign` has the same the same bit validity as `T`, so +// `T::is_bit_valid` is a sound implementation of `is_bit_valid`. +// +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + impl_or_verify!(T => Unaligned for Unalign); + impl_or_verify!(T: Immutable => Immutable for Unalign); + impl_or_verify!( + T: TryFromBytes => TryFromBytes for Unalign; + |c| T::is_bit_valid(c.transmute::<_, _, BecauseImmutable>()) + ); + impl_or_verify!(T: FromZeros => FromZeros for Unalign); + impl_or_verify!(T: FromBytes => FromBytes for Unalign); + impl_or_verify!(T: IntoBytes => IntoBytes for Unalign); +}; + +// Note that `Unalign: Clone` only if `T: Copy`. Since the inner `T` may not be +// aligned, there's no way to safely call `T::clone`, and so a `T: Clone` bound +// is not sufficient to implement `Clone` for `Unalign`. +impl Clone for Unalign { + #[inline(always)] + fn clone(&self) -> Unalign { + *self + } +} + +impl Unalign { + /// Constructs a new `Unalign`. + #[inline(always)] + pub const fn new(val: T) -> Unalign { + Unalign(val) + } + + /// Consumes `self`, returning the inner `T`. + #[inline(always)] + pub const fn into_inner(self) -> T { + // SAFETY: Since `Unalign` is `#[repr(C, packed)]`, it has the same size + // and bit validity as `T`. + // + // We do this instead of just destructuring in order to prevent + // `Unalign`'s `Drop::drop` from being run, since dropping is not + // supported in `const fn`s. + // + // FIXME(https://github.com/rust-lang/rust/issues/73255): Destructure + // instead of using unsafe. + unsafe { crate::util::transmute_unchecked(self) } + } + + /// Attempts to return a reference to the wrapped `T`, failing if `self` is + /// not properly aligned. + /// + /// If `self` does not satisfy `align_of::()`, then `try_deref` returns + /// `Err`. + /// + /// If `T: Unaligned`, then `Unalign` implements [`Deref`], and callers + /// may prefer [`Deref::deref`], which is infallible. + #[inline(always)] + pub fn try_deref(&self) -> Result<&T, AlignmentError<&Self, T>> { + let inner = Ptr::from_ref(self).transmute(); + match inner.try_into_aligned() { + Ok(aligned) => Ok(aligned.as_ref()), + Err(err) => Err(err.map_src( + #[inline(always)] + |src| src.into_unalign().as_ref(), + )), + } + } + + /// Attempts to return a mutable reference to the wrapped `T`, failing if + /// `self` is not properly aligned. + /// + /// If `self` does not satisfy `align_of::()`, then `try_deref` returns + /// `Err`. + /// + /// If `T: Unaligned`, then `Unalign` implements [`DerefMut`], and + /// callers may prefer [`DerefMut::deref_mut`], which is infallible. + #[inline(always)] + pub fn try_deref_mut(&mut self) -> Result<&mut T, AlignmentError<&mut Self, T>> { + let inner = Ptr::from_mut(self).transmute::<_, _, (_, (_, _))>(); + match inner.try_into_aligned() { + Ok(aligned) => Ok(aligned.as_mut()), + Err(err) => Err(err.map_src(|src| src.into_unalign().as_mut())), + } + } + + /// Returns a reference to the wrapped `T` without checking alignment. + /// + /// If `T: Unaligned`, then `Unalign` implements[ `Deref`], and callers + /// may prefer [`Deref::deref`], which is safe. + /// + /// # Safety + /// + /// The caller must guarantee that `self` satisfies `align_of::()`. + #[inline(always)] + pub const unsafe fn deref_unchecked(&self) -> &T { + // SAFETY: `Unalign` is `repr(transparent)`, so there is a valid `T` + // at the same memory location as `self`. It has no alignment guarantee, + // but the caller has promised that `self` is properly aligned, so we + // know that it is sound to create a reference to `T` at this memory + // location. + // + // We use `mem::transmute` instead of `&*self.get_ptr()` because + // dereferencing pointers is not stable in `const` on our current MSRV + // (1.56 as of this writing). + unsafe { mem::transmute(self) } + } + + /// Returns a mutable reference to the wrapped `T` without checking + /// alignment. + /// + /// If `T: Unaligned`, then `Unalign` implements[ `DerefMut`], and + /// callers may prefer [`DerefMut::deref_mut`], which is safe. + /// + /// # Safety + /// + /// The caller must guarantee that `self` satisfies `align_of::()`. + #[inline(always)] + pub unsafe fn deref_mut_unchecked(&mut self) -> &mut T { + // SAFETY: `self.get_mut_ptr()` returns a raw pointer to a valid `T` at + // the same memory location as `self`. It has no alignment guarantee, + // but the caller has promised that `self` is properly aligned, so we + // know that the pointer itself is aligned, and thus that it is sound to + // create a reference to a `T` at this memory location. + unsafe { &mut *self.get_mut_ptr() } + } + + /// Gets an unaligned raw pointer to the inner `T`. + /// + /// # Safety + /// + /// The returned raw pointer is not necessarily aligned to + /// `align_of::()`. Most functions which operate on raw pointers require + /// those pointers to be aligned, so calling those functions with the result + /// of `get_ptr` will result in undefined behavior if alignment is not + /// guaranteed using some out-of-band mechanism. In general, the only + /// functions which are safe to call with this pointer are those which are + /// explicitly documented as being sound to use with an unaligned pointer, + /// such as [`read_unaligned`]. + /// + /// Even if the caller is permitted to mutate `self` (e.g. they have + /// ownership or a mutable borrow), it is not guaranteed to be sound to + /// write through the returned pointer. If writing is required, prefer + /// [`get_mut_ptr`] instead. + /// + /// [`read_unaligned`]: core::ptr::read_unaligned + /// [`get_mut_ptr`]: Unalign::get_mut_ptr + #[inline(always)] + pub const fn get_ptr(&self) -> *const T { + ptr::addr_of!(self.0) + } + + /// Gets an unaligned mutable raw pointer to the inner `T`. + /// + /// # Safety + /// + /// The returned raw pointer is not necessarily aligned to + /// `align_of::()`. Most functions which operate on raw pointers require + /// those pointers to be aligned, so calling those functions with the result + /// of `get_ptr` will result in undefined behavior if alignment is not + /// guaranteed using some out-of-band mechanism. In general, the only + /// functions which are safe to call with this pointer are those which are + /// explicitly documented as being sound to use with an unaligned pointer, + /// such as [`read_unaligned`]. + /// + /// [`read_unaligned`]: core::ptr::read_unaligned + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn get_mut_ptr(&mut self) -> *mut T { + ptr::addr_of_mut!(self.0) + } + + /// Sets the inner `T`, dropping the previous value. + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn set(&mut self, t: T) { + *self = Unalign::new(t); + } + + /// Updates the inner `T` by calling a function on it. + /// + /// If [`T: Unaligned`], then `Unalign` implements [`DerefMut`], and that + /// impl should be preferred over this method when performing updates, as it + /// will usually be faster and more ergonomic. + /// + /// For large types, this method may be expensive, as it requires copying + /// `2 * size_of::()` bytes. \[1\] + /// + /// \[1\] Since the inner `T` may not be aligned, it would not be sound to + /// invoke `f` on it directly. Instead, `update` moves it into a + /// properly-aligned location in the local stack frame, calls `f` on it, and + /// then moves it back to its original location in `self`. + /// + /// [`T: Unaligned`]: Unaligned + #[inline] + pub fn update O>(&mut self, f: F) -> O { + if mem::align_of::() == 1 { + // While we advise callers to use `DerefMut` when `T: Unaligned`, + // not all callers will be able to guarantee `T: Unaligned` in all + // cases. In particular, callers who are themselves providing an API + // which is generic over `T` may sometimes be called by *their* + // callers with `T` such that `align_of::() == 1`, but cannot + // guarantee this in the general case. Thus, this optimization may + // sometimes be helpful. + + // SAFETY: Since `T`'s alignment is 1, `self` satisfies its + // alignment by definition. + let t = unsafe { self.deref_mut_unchecked() }; + return f(t); + } + + // On drop, this moves `copy` out of itself and uses `ptr::write` to + // overwrite `slf`. + struct WriteBackOnDrop { + copy: ManuallyDrop, + slf: *mut Unalign, + } + + impl Drop for WriteBackOnDrop { + fn drop(&mut self) { + // SAFETY: We never use `copy` again as required by + // `ManuallyDrop::take`. + let copy = unsafe { ManuallyDrop::take(&mut self.copy) }; + // SAFETY: `slf` is the raw pointer value of `self`. We know it + // is valid for writes and properly aligned because `self` is a + // mutable reference, which guarantees both of these properties. + unsafe { ptr::write(self.slf, Unalign::new(copy)) }; + } + } + + // SAFETY: We know that `self` is valid for reads, properly aligned, and + // points to an initialized `Unalign` because it is a mutable + // reference, which guarantees all of these properties. + // + // Since `T: !Copy`, it would be unsound in the general case to allow + // both the original `Unalign` and the copy to be used by safe code. + // We guarantee that the copy is used to overwrite the original in the + // `Drop::drop` impl of `WriteBackOnDrop`. So long as this `drop` is + // called before any other safe code executes, soundness is upheld. + // While this method can terminate in two ways (by returning normally or + // by unwinding due to a panic in `f`), in both cases, `write_back` is + // dropped - and its `drop` called - before any other safe code can + // execute. + let copy = unsafe { ptr::read(self) }.into_inner(); + let mut write_back = WriteBackOnDrop { copy: ManuallyDrop::new(copy), slf: self }; + + let ret = f(&mut write_back.copy); + + drop(write_back); + ret + } +} + +impl Unalign { + /// Gets a copy of the inner `T`. + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn get(&self) -> T { + let Unalign(val) = *self; + val + } +} + +impl Deref for Unalign { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + Ptr::from_ref(self).transmute().bikeshed_recall_aligned().as_ref() + } +} + +impl DerefMut for Unalign { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + Ptr::from_mut(self).transmute::<_, _, (_, (_, _))>().bikeshed_recall_aligned().as_mut() + } +} + +impl PartialOrd> for Unalign { + #[inline(always)] + fn partial_cmp(&self, other: &Unalign) -> Option { + PartialOrd::partial_cmp(self.deref(), other.deref()) + } +} + +impl Ord for Unalign { + #[inline(always)] + fn cmp(&self, other: &Unalign) -> Ordering { + Ord::cmp(self.deref(), other.deref()) + } +} + +impl PartialEq> for Unalign { + #[inline(always)] + fn eq(&self, other: &Unalign) -> bool { + PartialEq::eq(self.deref(), other.deref()) + } +} + +impl Eq for Unalign {} + +impl Hash for Unalign { + #[inline(always)] + fn hash(&self, state: &mut H) + where + H: Hasher, + { + self.deref().hash(state); + } +} + +impl Debug for Unalign { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Debug::fmt(self.deref(), f) + } +} + +impl Display for Unalign { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(self.deref(), f) + } +} + +/// A wrapper type to construct uninitialized instances of `T`. +/// +/// `MaybeUninit` is identical to the [standard library +/// `MaybeUninit`][core-maybe-uninit] type except that it supports unsized +/// types. +/// +/// # Layout +/// +/// The same layout guarantees and caveats apply to `MaybeUninit` as apply to +/// the [standard library `MaybeUninit`][core-maybe-uninit] with one exception: +/// for `T: !Sized`, there is no single value for `T`'s size. Instead, for such +/// types, the following are guaranteed: +/// - Every [valid size][valid-size] for `T` is a valid size for +/// `MaybeUninit` and vice versa +/// - Given `t: *const T` and `m: *const MaybeUninit` with identical fat +/// pointer metadata, `t` and `m` address the same number of bytes (and +/// likewise for `*mut`) +/// +/// [core-maybe-uninit]: core::mem::MaybeUninit +/// [valid-size]: crate::KnownLayout#what-is-a-valid-size +#[repr(transparent)] +#[doc(hidden)] +pub struct MaybeUninit( + // SAFETY: `MaybeUninit` has the same size as `T`, because (by invariant + // on `T::MaybeUninit`) `T::MaybeUninit` has `T::LAYOUT` identical to `T`, + // and because (invariant on `T::LAYOUT`) we can trust that `LAYOUT` + // accurately reflects the layout of `T`. By invariant on `T::MaybeUninit`, + // it admits uninitialized bytes in all positions. Because `MaybeUninit` is + // marked `repr(transparent)`, these properties additionally hold true for + // `Self`. + T::MaybeUninit, +); + +#[doc(hidden)] +impl MaybeUninit { + /// Constructs a `MaybeUninit` initialized with the given value. + #[inline(always)] + pub fn new(val: T) -> Self + where + T: Sized, + Self: Sized, + { + // SAFETY: It is valid to transmute `val` to `MaybeUninit` because it + // is both valid to transmute `val` to `T::MaybeUninit`, and it is valid + // to transmute from `T::MaybeUninit` to `MaybeUninit`. + // + // First, it is valid to transmute `val` to `T::MaybeUninit` because, by + // invariant on `T::MaybeUninit`: + // - For `T: Sized`, `T` and `T::MaybeUninit` have the same size. + // - All byte sequences of the correct size are valid values of + // `T::MaybeUninit`. + // + // Second, it is additionally valid to transmute from `T::MaybeUninit` + // to `MaybeUninit`, because `MaybeUninit` is a + // `repr(transparent)` wrapper around `T::MaybeUninit`. + // + // These two transmutes are collapsed into one so we don't need to add a + // `T::MaybeUninit: Sized` bound to this function's `where` clause. + unsafe { crate::util::transmute_unchecked(val) } + } + + /// Constructs an uninitialized `MaybeUninit`. + #[must_use] + #[inline(always)] + pub fn uninit() -> Self + where + T: Sized, + Self: Sized, + { + let uninit = CoreMaybeUninit::::uninit(); + // SAFETY: It is valid to transmute from `CoreMaybeUninit` to + // `MaybeUninit` since they both admit uninitialized bytes in all + // positions, and they have the same size (i.e., that of `T`). + // + // `MaybeUninit` has the same size as `T`, because (by invariant on + // `T::MaybeUninit`) `T::MaybeUninit` has `T::LAYOUT` identical to `T`, + // and because (invariant on `T::LAYOUT`) we can trust that `LAYOUT` + // accurately reflects the layout of `T`. + // + // `CoreMaybeUninit` has the same size as `T` [1] and admits + // uninitialized bytes in all positions. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit` is guaranteed to have the same size, alignment, + // and ABI as `T` + unsafe { crate::util::transmute_unchecked(uninit) } + } + + /// Creates a `Box>`. + /// + /// This function is useful for allocating large, uninit values on the heap + /// without ever creating a temporary instance of `Self` on the stack. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is guaranteed + /// never to cause a panic or an abort. + #[cfg(feature = "alloc")] + #[inline] + pub fn new_boxed_uninit(meta: T::PointerMetadata) -> Result, AllocError> { + // SAFETY: `alloc::alloc::alloc_zeroed` is a valid argument of + // `new_box`. The referent of the pointer returned by `alloc` (and, + // consequently, the `Box` derived from it) is a valid instance of + // `Self`, because `Self` is `MaybeUninit` and thus admits arbitrary + // (un)initialized bytes. + unsafe { crate::util::new_box(meta, alloc::alloc::alloc) } + } + + /// Extracts the value from the `MaybeUninit` container. + /// + /// # Safety + /// + /// The caller must ensure that `self` is in an bit-valid state. Depending + /// on subsequent use, it may also need to be in a library-valid state. + #[inline(always)] + pub unsafe fn assume_init(self) -> T + where + T: Sized, + Self: Sized, + { + // SAFETY: The caller guarantees that `self` is in an bit-valid state. + unsafe { crate::util::transmute_unchecked(self) } + } +} + +impl fmt::Debug for MaybeUninit { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(core::any::type_name::()) + } +} + +#[allow(unreachable_pub)] // False positive on MSRV +#[doc(hidden)] +pub use read_only_def::*; +mod read_only_def { + /// A read-only wrapper. + /// + /// A `ReadOnly` disables any interior mutability in `T`, ensuring that + /// a `&ReadOnly` is genuinely read-only. Thus, `ReadOnly` is + /// [`Immutable`] regardless of whether `T` is. + /// + /// Note that `&mut ReadOnly` still permits mutation – the read-only + /// property only applies to shared references. + /// + /// [`Immutable`]: crate::Immutable + #[repr(transparent)] + pub struct ReadOnly { + // INVARIANT: `inner` is never mutated through a `&ReadOnly` + // reference. + inner: T, + } + + impl ReadOnly { + /// Creates a new `ReadOnly`. + #[must_use] + #[inline(always)] + pub const fn new(t: T) -> ReadOnly { + ReadOnly { inner: t } + } + + /// Returns the inner value. + #[must_use] + #[inline(always)] + pub fn into_inner(r: ReadOnly) -> T { + r.inner + } + } + + impl ReadOnly { + #[inline(always)] + pub(crate) fn as_mut(r: &mut ReadOnly) -> &mut T { + // SAFETY: `r: &mut ReadOnly`, so this doesn't violate the invariant + // that `inner` is never mutated through a `&ReadOnly` reference. + &mut r.inner + } + + /// # Safety + /// + /// The caller promises not to mutate the referent (i.e., via interior + /// mutation). + pub(crate) const unsafe fn as_ref_unchecked(r: &ReadOnly) -> &T { + // SAFETY: The caller promises not to mutate the referent. + &r.inner + } + } +} + +// SAFETY: `ReadOnly` is a `#[repr(transparent)` wrapper around `T`. +const _: () = unsafe { + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] ReadOnly); +}; + +#[allow(clippy::multiple_unsafe_ops_per_block)] +// SAFETY: +// - `ReadOnly` has the same alignment as `T`, and so it is `Unaligned` +// exactly when `T` is as well. +// - `ReadOnly` has the same bit validity as `T`, and so this `is_bit_valid` +// implementation is correct, and thus the `TryFromBytes` impl is sound. +// - `ReadOnly` has the same bit validity as `T`, and so it is `FromZeros`, +// `FromBytes`, and `IntoBytes` exactly when `T` is as well. +const _: () = unsafe { + unsafe_impl!(T: ?Sized + Unaligned => Unaligned for ReadOnly); + unsafe_impl!( + T: ?Sized + TryFromBytes => TryFromBytes for ReadOnly; + |c| T::is_bit_valid(c.cast::<_, as SizeEq>>>::CastFrom, _>()) + ); + unsafe_impl!(T: ?Sized + FromZeros => FromZeros for ReadOnly); + unsafe_impl!(T: ?Sized + FromBytes => FromBytes for ReadOnly); + unsafe_impl!(T: ?Sized + IntoBytes => IntoBytes for ReadOnly); +}; + +// SAFETY: By invariant, `inner` is never mutated through a `&ReadOnly` +// reference. +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for ReadOnly); +}; + +const _: () = { + use crate::pointer::cast::CastExact; + + // SAFETY: `ReadOnly` has the same layout as `T`. + define_cast!(unsafe { pub CastFromReadOnly = ReadOnly => T}); + // SAFETY: `ReadOnly` has the same layout as `T`. + unsafe impl CastExact, T> for CastFromReadOnly {} + // SAFETY: `ReadOnly` has the same layout as `T`. + define_cast!(unsafe { pub CastToReadOnly = T => ReadOnly}); + // SAFETY: `ReadOnly` has the same layout as `T`. + unsafe impl CastExact> for CastToReadOnly {} + + impl SizeEq> for T { + type CastFrom = CastFromReadOnly; + } + + impl SizeEq for ReadOnly { + type CastFrom = CastToReadOnly; + } +}; + +// SAFETY: `ReadOnly` is a `#[repr(transparent)]` wrapper around `T`, and so +// it has the same bit validity as `T`. +unsafe impl TransmuteFrom for ReadOnly {} + +// SAFETY: `ReadOnly` is a `#[repr(transparent)]` wrapper around `T`, and so +// it has the same bit validity as `T`. +unsafe impl TransmuteFrom, Valid, Valid> for T {} + +impl<'a, T: ?Sized + Immutable> From<&'a T> for &'a ReadOnly { + #[inline(always)] + fn from(t: &'a T) -> &'a ReadOnly { + let ro = Ptr::from_ref(t).transmute::<_, _, (_, _)>(); + // SAFETY: `ReadOnly` has the same alignment as `T`, and + // `Ptr::from_ref` produces an aligned `Ptr`. + let ro = unsafe { ro.assume_alignment() }; + ro.as_ref() + } +} + +impl Deref for ReadOnly { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + // SAFETY: By `T: Immutable`, `&T` doesn't permit interior mutation. + unsafe { ReadOnly::as_ref_unchecked(self) } + } +} + +impl DerefMut for ReadOnly { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + ReadOnly::as_mut(self) + } +} + +impl Debug for ReadOnly { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.deref().fmt(f) + } +} + +// SAFETY: See safety comment on `ProjectToTag`. +unsafe impl HasTag for ReadOnly { + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Tag = T::Tag; + + // SAFETY: `>>::CastFrom` is a no-op projection that + // produces a pointer with the same referent. By invariant, for any `Ptr<'_, + // T, I>` it is sound to use `T::ProjectToTag` to project to a `Ptr<'_, + // T::Tag, I>`. Since `ReadOnly` has the same layout and validity as `T`, + // the same is true of projecting from a `Ptr<'_, ReadOnly, I>`. + type ProjectToTag = crate::pointer::cast::TransitiveProject< + T, + >>::CastFrom, + T::ProjectToTag, + >; +} + +// SAFETY: `ReadOnly` is a `#[repr(transparent)]` wrapper around `T`, and so +// has the same fields at the same offsets. Thus, it satisfies the safety +// invariants of `HasField` for field `f` exactly +// when `T` does, as guaranteed by the `T: HasField` bound: +// - If `VARIANT_ID` is `STRUCT_VARIANT_ID` or `UNION_VARIANT_ID`, then `T` has +// the layout of a struct or union type. Since `ReadOnly` is a transparent +// wrapper around `T`, it does too. Otherwise, if `VARIANT_ID` is an enum +// variant index, then `T` has the layout of an enum type, and `ReadOnly` +// does too. +// - By `T: HasField<_, _, FIELD_ID>`: +// - `T` has a field `f` with name `n` such that +// `FIELD_ID = zerocopy::ident_id!(n)` or at index `i` such that +// `FIELD_ID = zerocopy::ident_id!(i)`. +// - `Field` has the same visibility as `f`. +// - `T::Type` has the same type as `f`. Thus, `ReadOnly` has the +// same type as `f`, wrapped in `ReadOnly`. +// +// `project` satisfies its post-condition – namely, that the returned pointer +// refers to a non-strict subset of the bytes of `slf`'s referent, and has the +// same provenance as `slf` – because all intermediate operations satisfy those +// same conditions. +unsafe impl + HasField for ReadOnly +where + T: HasField + ?Sized, +{ + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Type = ReadOnly; + + #[inline(always)] + fn project(slf: PtrInner<'_, Self>) -> *mut ReadOnly { + slf.project::<_, >>::CastFrom>() + .project::<_, crate::pointer::cast::Projection>() + .project::<_, as SizeEq>::CastFrom>() + .as_non_null() + .as_ptr() + } +} + +// SAFETY: `ReadOnly` is a `#[repr(transparent)]` wrapper around `T`, and so +// has the same fields at the same offsets. `is_projectable` simply delegates to +// `T::is_projectable`, which is sound because a `Ptr<'_, ReadOnly, I>` will +// be projectable exactly when a `Ptr<'_, T, I>` referent is. +unsafe impl + ProjectField for ReadOnly +where + T: ProjectField + ?Sized, + I: invariant::Invariants, +{ + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Invariants = T::Invariants; + + type Error = T::Error; + + #[inline(always)] + fn is_projectable<'a>(ptr: Ptr<'a, Self::Tag, I>) -> Result<(), Self::Error> { + T::is_projectable(ptr) + } +} + +#[cfg(test)] +mod tests { + use core::panic::AssertUnwindSafe; + + use super::*; + use crate::util::testutil::*; + + #[test] + fn test_unalign() { + // Test methods that don't depend on alignment. + let mut u = Unalign::new(AU64(123)); + assert_eq!(u.get(), AU64(123)); + assert_eq!(u.into_inner(), AU64(123)); + assert_eq!(u.get_ptr(), <*const _>::cast::(&u)); + assert_eq!(u.get_mut_ptr(), <*mut _>::cast::(&mut u)); + u.set(AU64(321)); + assert_eq!(u.get(), AU64(321)); + + // Test methods that depend on alignment (when alignment is satisfied). + let mut u: Align<_, AU64> = Align::new(Unalign::new(AU64(123))); + assert_eq!(u.t.try_deref().unwrap(), &AU64(123)); + assert_eq!(u.t.try_deref_mut().unwrap(), &mut AU64(123)); + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + assert_eq!(unsafe { u.t.deref_unchecked() }, &AU64(123)); + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + assert_eq!(unsafe { u.t.deref_mut_unchecked() }, &mut AU64(123)); + *u.t.try_deref_mut().unwrap() = AU64(321); + assert_eq!(u.t.get(), AU64(321)); + + // Test methods that depend on alignment (when alignment is not + // satisfied). + let mut u: ForceUnalign<_, AU64> = ForceUnalign::new(Unalign::new(AU64(123))); + assert!(matches!(u.t.try_deref(), Err(AlignmentError { .. }))); + assert!(matches!(u.t.try_deref_mut(), Err(AlignmentError { .. }))); + + // Test methods that depend on `T: Unaligned`. + let mut u = Unalign::new(123u8); + assert_eq!(u.try_deref(), Ok(&123)); + assert_eq!(u.try_deref_mut(), Ok(&mut 123)); + assert_eq!(u.deref(), &123); + assert_eq!(u.deref_mut(), &mut 123); + *u = 21; + assert_eq!(u.get(), 21); + + // Test that some `Unalign` functions and methods are `const`. + const _UNALIGN: Unalign = Unalign::new(0); + const _UNALIGN_PTR: *const u64 = _UNALIGN.get_ptr(); + const _U64: u64 = _UNALIGN.into_inner(); + // Make sure all code is considered "used". + // + // FIXME(https://github.com/rust-lang/rust/issues/104084): Remove this + // attribute. + #[allow(dead_code)] + const _: () = { + let x: Align<_, AU64> = Align::new(Unalign::new(AU64(123))); + // Make sure that `deref_unchecked` is `const`. + // + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + let au64 = unsafe { x.t.deref_unchecked() }; + match au64 { + AU64(123) => {} + _ => const_unreachable!(), + } + }; + } + + #[test] + fn test_unalign_update() { + let mut u = Unalign::new(AU64(123)); + u.update(|a| a.0 += 1); + assert_eq!(u.get(), AU64(124)); + + // Test that, even if the callback panics, the original is still + // correctly overwritten. Use a `Box` so that Miri is more likely to + // catch any unsoundness (which would likely result in two `Box`es for + // the same heap object, which is the sort of thing that Miri would + // probably catch). + let mut u = Unalign::new(Box::new(AU64(123))); + let res = std::panic::catch_unwind(AssertUnwindSafe(|| { + u.update(|a| { + a.0 += 1; + panic!(); + }) + })); + assert!(res.is_err()); + assert_eq!(u.into_inner(), Box::new(AU64(124))); + + // Test the align_of::() == 1 optimization. + let mut u = Unalign::new([0u8, 1]); + u.update(|a| a[0] += 1); + assert_eq!(u.get(), [1u8, 1]); + } + + #[test] + fn test_unalign_copy_clone() { + // Test that `Copy` and `Clone` do not cause soundness issues. This test + // is mainly meant to exercise UB that would be caught by Miri. + + // `u.t` is definitely not validly-aligned for `AU64`'s alignment of 8. + let u = ForceUnalign::<_, AU64>::new(Unalign::new(AU64(123))); + #[allow(clippy::clone_on_copy)] + let v = u.t.clone(); + let w = u.t; + assert_eq!(u.t.get(), v.get()); + assert_eq!(u.t.get(), w.get()); + assert_eq!(v.get(), w.get()); + } + + #[test] + fn test_unalign_trait_impls() { + let zero = Unalign::new(0u8); + let one = Unalign::new(1u8); + + assert!(zero < one); + assert_eq!(PartialOrd::partial_cmp(&zero, &one), Some(Ordering::Less)); + assert_eq!(Ord::cmp(&zero, &one), Ordering::Less); + + assert_ne!(zero, one); + assert_eq!(zero, zero); + assert!(!PartialEq::eq(&zero, &one)); + assert!(PartialEq::eq(&zero, &zero)); + + fn hash(t: &T) -> u64 { + let mut h = std::collections::hash_map::DefaultHasher::new(); + t.hash(&mut h); + h.finish() + } + + assert_eq!(hash(&zero), hash(&0u8)); + assert_eq!(hash(&one), hash(&1u8)); + + assert_eq!(format!("{:?}", zero), format!("{:?}", 0u8)); + assert_eq!(format!("{:?}", one), format!("{:?}", 1u8)); + assert_eq!(format!("{}", zero), format!("{}", 0u8)); + assert_eq!(format!("{}", one), format!("{}", 1u8)); + } + + #[test] + #[allow(clippy::as_conversions)] + fn test_maybe_uninit() { + // int + { + let input = 42; + let uninit = MaybeUninit::new(input); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(input, output); + } + + // thin ref + { + let input = 42; + let uninit = MaybeUninit::new(&input); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(&input as *const _, output as *const _); + assert_eq!(input, *output); + } + + // wide ref + { + let input = [1, 2, 3, 4]; + let uninit = MaybeUninit::new(&input[..]); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(&input[..] as *const _, output as *const _); + assert_eq!(input, *output); + } + } + #[test] + fn test_maybe_uninit_uninit() { + let _uninit = MaybeUninit::::uninit(); + // Cannot check value, but can check it compiles and runs + } + + #[test] + #[cfg(feature = "alloc")] + fn test_maybe_uninit_new_boxed_uninit() { + let _boxed = MaybeUninit::::new_boxed_uninit(()).unwrap(); + } + + #[test] + fn test_maybe_uninit_debug() { + let uninit = MaybeUninit::::uninit(); + assert!(format!("{:?}", uninit).contains("MaybeUninit")); + } +} -- 2.54.0