Add const UTF-8 to UTF-16 conversion macros

`wide_str!` creates a null terminated UTF-16 string whereas `utf16!` just creates a UTF-16 string without adding a null.

Author: ChrisDenton

library/std/src/sys/pal/windows/api.rs

@@ -34,6 +34,105 @@ use core::ptr::addr_of;
 
 use super::c;
 
+/// Creates a null-terminated UTF-16 string from a str.
+macro_rules! wide_str {
+    ($str:literal) => {{
+        const _: () = {
+            if core::slice::memchr::memchr(0, $str.as_bytes()).is_some() {
+                panic!("null terminated strings cannot contain interior nulls");
+            }
+        };
+        utf16!(concat!($str, '\0'))
+    }};
+}
+
+/// Creates a UTF-16 string from a str without null termination.
+macro_rules! utf16 {
+    // Note: this macro uses triple underscores to avoid const cycles
+    ($str:expr) => {{
+        const ___UTF8: &str = $str;
+        const ___UTF16_LEN: usize = crate::sys::pal::windows::api::utf16_len(___UTF8);
+        const ___UTF16: [u16; ___UTF16_LEN] = crate::sys::pal::windows::api::to_utf16(___UTF8);
+        &___UTF16
+    }};
+}
+
+#[cfg(test)]
+mod tests;
+
+/// Gets the UTF-16 length of a UTF-8 string, for use in the wide_str macro.
+pub const fn utf16_len(s: &str) -> usize {
+    let s = s.as_bytes();
+    let mut i = 0;
+    let mut len = 0;
+    while i < s.len() {
+        // the length of a UTF-8 encoded code-point is given by the number of
+        // leading ones, except in the case of ASCII.
+        let utf8_len = match s[i].leading_ones() {
+            0 => 1,
+            n => n as usize,
+        };
+        i += utf8_len;
+        len += if utf8_len < 4 { 1 } else { 2 };
+    }
+    len
+}
+
+/// Const convert UTF-8 to UTF-16, for use in the wide_str macro.
+///
+/// Note that this is designed for use in const contexts so is not optimized.
+pub const fn to_utf16<const UTF16_LEN: usize>(s: &str) -> [u16; UTF16_LEN] {
+    let mut output = [0_u16; UTF16_LEN];
+    let mut pos = 0;
+    let s = s.as_bytes();
+    let mut i = 0;
+    while i < s.len() {
+        match s[i].leading_ones() {
+            // Decode UTF-8 based on its length.
+            // See https://en.wikipedia.org/wiki/UTF-8
+            0 => {
+                // ASCII is the same in both encodings
+                output[pos] = s[i] as u16;
+                i += 1;
+                pos += 1;
+            }
+            2 => {
+                // Bits: 110xxxxx 10xxxxxx
+                output[pos] = ((s[i] as u16 & 0b11111) << 6) | (s[i + 1] as u16 & 0b111111);
+                i += 2;
+                pos += 1;
+            }
+            3 => {
+                // Bits: 1110xxxx 10xxxxxx 10xxxxxx
+                output[pos] = ((s[i] as u16 & 0b1111) << 12)
+                    | ((s[i + 1] as u16 & 0b111111) << 6)
+                    | (s[i + 2] as u16 & 0b111111);
+                i += 3;
+                pos += 1;
+            }
+            4 => {
+                // Bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                let mut c = ((s[i] as u32 & 0b111) << 18)
+                    | ((s[i + 1] as u32 & 0b111111) << 12)
+                    | ((s[i + 2] as u32 & 0b111111) << 6)
+                    | (s[i + 3] as u32 & 0b111111);
+                // re-encode as UTF-16 (see https://en.wikipedia.org/wiki/UTF-16)
+                // - Subtract 0x10000 from the code point
+                // - For the high surrogate, shift right by 10 then add 0xD800
+                // - For the low surrogate, take the low 10 bits then add 0xDC00
+                c -= 0x10000;
+                output[pos] = ((c >> 10) + 0xD800) as u16;
+                output[pos + 1] = ((c & 0b1111111111) + 0xDC00) as u16;
+                i += 4;
+                pos += 2;
+            }
+            // valid UTF-8 cannot have any other values
+            _ => unreachable!(),
+        }
+    }
+    output
+}
+
 /// Helper method for getting the size of `T` as a u32.
 /// Errors at compile time if the size would overflow.
 ///

library/std/src/sys/pal/windows/api/tests.rs

@@ -0,0 +1,14 @@
+macro_rules! check_utf16 {
+    ($str:literal) => {{
+        assert!(wide_str!($str).iter().copied().eq($str.encode_utf16().chain([0])));
+        assert!(utf16!($str).iter().copied().eq($str.encode_utf16()));
+    }};
+}
+
+#[test]
+fn test_utf16_macros() {
+    check_utf16!("hello world");
+    check_utf16!("€4.50");
+    check_utf16!("𨉟呐㗂越");
+    check_utf16!("Pchnąć w tę łódź jeża lub ośm skrzyń fig");
+}

library/std/src/sys/pal/windows/mod.rs

@@ -12,6 +12,9 @@ pub use self::rand::hashmap_random_keys;
 #[macro_use]
 pub mod compat;
 
+#[macro_use]
+mod api;
+
 pub mod alloc;
 pub mod args;
 pub mod c;
@@ -41,8 +44,6 @@ cfg_if::cfg_if! {
     }
 }
 
-mod api;
-
 /// Map a Result<T, WinError> to io::Result<T>.
 trait IoResult<T> {
     fn io_result(self) -> crate::io::Result<T>;