RPM build fix (reverted CI changes which will need to be un-reverted or made conditional) and vendor Rust dependencies to make builds much faster in any CI system.

zeroidc/vendor/base64/src/read/decoder.rs

@@ -0,0 +1,282 @@
+use crate::{decode_config_slice, Config, DecodeError};
+use std::io::Read;
+use std::{cmp, fmt, io};
+
+// This should be large, but it has to fit on the stack.
+pub(crate) const BUF_SIZE: usize = 1024;
+
+// 4 bytes of base64 data encode 3 bytes of raw data (modulo padding).
+const BASE64_CHUNK_SIZE: usize = 4;
+const DECODED_CHUNK_SIZE: usize = 3;
+
+/// A `Read` implementation that decodes base64 data read from an underlying reader.
+///
+/// # Examples
+///
+/// ```
+/// use std::io::Read;
+/// use std::io::Cursor;
+///
+/// // use a cursor as the simplest possible `Read` -- in real code this is probably a file, etc.
+/// let mut wrapped_reader = Cursor::new(b"YXNkZg==");
+/// let mut decoder = base64::read::DecoderReader::new(
+///     &mut wrapped_reader, base64::STANDARD);
+///
+/// // handle errors as you normally would
+/// let mut result = Vec::new();
+/// decoder.read_to_end(&mut result).unwrap();
+///
+/// assert_eq!(b"asdf", &result[..]);
+///
+/// ```
+pub struct DecoderReader<'a, R: 'a + io::Read> {
+    config: Config,
+    /// Where b64 data is read from
+    r: &'a mut R,
+
+    // Holds b64 data read from the delegate reader.
+    b64_buffer: [u8; BUF_SIZE],
+    // The start of the pending buffered data in b64_buffer.
+    b64_offset: usize,
+    // The amount of buffered b64 data.
+    b64_len: usize,
+    // Since the caller may provide us with a buffer of size 1 or 2 that's too small to copy a
+    // decoded chunk in to, we have to be able to hang on to a few decoded bytes.
+    // Technically we only need to hold 2 bytes but then we'd need a separate temporary buffer to
+    // decode 3 bytes into and then juggle copying one byte into the provided read buf and the rest
+    // into here, which seems like a lot of complexity for 1 extra byte of storage.
+    decoded_buffer: [u8; 3],
+    // index of start of decoded data
+    decoded_offset: usize,
+    // length of decoded data
+    decoded_len: usize,
+    // used to provide accurate offsets in errors
+    total_b64_decoded: usize,
+}
+
+impl<'a, R: io::Read> fmt::Debug for DecoderReader<'a, R> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_struct("DecoderReader")
+            .field("config", &self.config)
+            .field("b64_offset", &self.b64_offset)
+            .field("b64_len", &self.b64_len)
+            .field("decoded_buffer", &self.decoded_buffer)
+            .field("decoded_offset", &self.decoded_offset)
+            .field("decoded_len", &self.decoded_len)
+            .field("total_b64_decoded", &self.total_b64_decoded)
+            .finish()
+    }
+}
+
+impl<'a, R: io::Read> DecoderReader<'a, R> {
+    /// Create a new decoder that will read from the provided reader `r`.
+    pub fn new(r: &'a mut R, config: Config) -> Self {
+        DecoderReader {
+            config,
+            r,
+            b64_buffer: [0; BUF_SIZE],
+            b64_offset: 0,
+            b64_len: 0,
+            decoded_buffer: [0; DECODED_CHUNK_SIZE],
+            decoded_offset: 0,
+            decoded_len: 0,
+            total_b64_decoded: 0,
+        }
+    }
+
+    /// Write as much as possible of the decoded buffer into the target buffer.
+    /// Must only be called when there is something to write and space to write into.
+    /// Returns a Result with the number of (decoded) bytes copied.
+    fn flush_decoded_buf(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        debug_assert!(self.decoded_len > 0);
+        debug_assert!(buf.len() > 0);
+
+        let copy_len = cmp::min(self.decoded_len, buf.len());
+        debug_assert!(copy_len > 0);
+        debug_assert!(copy_len <= self.decoded_len);
+
+        buf[..copy_len].copy_from_slice(
+            &self.decoded_buffer[self.decoded_offset..self.decoded_offset + copy_len],
+        );
+
+        self.decoded_offset += copy_len;
+        self.decoded_len -= copy_len;
+
+        debug_assert!(self.decoded_len < DECODED_CHUNK_SIZE);
+
+        Ok(copy_len)
+    }
+
+    /// Read into the remaining space in the buffer after the current contents.
+    /// Must only be called when there is space to read into in the buffer.
+    /// Returns the number of bytes read.
+    fn read_from_delegate(&mut self) -> io::Result<usize> {
+        debug_assert!(self.b64_offset + self.b64_len < BUF_SIZE);
+
+        let read = self
+            .r
+            .read(&mut self.b64_buffer[self.b64_offset + self.b64_len..])?;
+        self.b64_len += read;
+
+        debug_assert!(self.b64_offset + self.b64_len <= BUF_SIZE);
+
+        return Ok(read);
+    }
+
+    /// Decode the requested number of bytes from the b64 buffer into the provided buffer. It's the
+    /// caller's responsibility to choose the number of b64 bytes to decode correctly.
+    ///
+    /// Returns a Result with the number of decoded bytes written to `buf`.
+    fn decode_to_buf(&mut self, num_bytes: usize, buf: &mut [u8]) -> io::Result<usize> {
+        debug_assert!(self.b64_len >= num_bytes);
+        debug_assert!(self.b64_offset + self.b64_len <= BUF_SIZE);
+        debug_assert!(buf.len() > 0);
+
+        let decoded = decode_config_slice(
+            &self.b64_buffer[self.b64_offset..self.b64_offset + num_bytes],
+            self.config,
+            &mut buf[..],
+        )
+        .map_err(|e| match e {
+            DecodeError::InvalidByte(offset, byte) => {
+                DecodeError::InvalidByte(self.total_b64_decoded + offset, byte)
+            }
+            DecodeError::InvalidLength => DecodeError::InvalidLength,
+            DecodeError::InvalidLastSymbol(offset, byte) => {
+                DecodeError::InvalidLastSymbol(self.total_b64_decoded + offset, byte)
+            }
+        })
+        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+
+        self.total_b64_decoded += num_bytes;
+        self.b64_offset += num_bytes;
+        self.b64_len -= num_bytes;
+
+        debug_assert!(self.b64_offset + self.b64_len <= BUF_SIZE);
+
+        Ok(decoded)
+    }
+}
+
+impl<'a, R: Read> Read for DecoderReader<'a, R> {
+    /// Decode input from the wrapped reader.
+    ///
+    /// Under non-error circumstances, this returns `Ok` with the value being the number of bytes
+    /// written in `buf`.
+    ///
+    /// Where possible, this function buffers base64 to minimize the number of read() calls to the
+    /// delegate reader.
+    ///
+    /// # Errors
+    ///
+    /// Any errors emitted by the delegate reader are returned. Decoding errors due to invalid
+    /// base64 are also possible, and will have `io::ErrorKind::InvalidData`.
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        if buf.len() == 0 {
+            return Ok(0);
+        }
+
+        // offset == BUF_SIZE when we copied it all last time
+        debug_assert!(self.b64_offset <= BUF_SIZE);
+        debug_assert!(self.b64_offset + self.b64_len <= BUF_SIZE);
+        debug_assert!(if self.b64_offset == BUF_SIZE {
+            self.b64_len == 0
+        } else {
+            self.b64_len <= BUF_SIZE
+        });
+
+        debug_assert!(if self.decoded_len == 0 {
+            // can be = when we were able to copy the complete chunk
+            self.decoded_offset <= DECODED_CHUNK_SIZE
+        } else {
+            self.decoded_offset < DECODED_CHUNK_SIZE
+        });
+
+        // We shouldn't ever decode into here when we can't immediately write at least one byte into
+        // the provided buf, so the effective length should only be 3 momentarily between when we
+        // decode and when we copy into the target buffer.
+        debug_assert!(self.decoded_len < DECODED_CHUNK_SIZE);
+        debug_assert!(self.decoded_len + self.decoded_offset <= DECODED_CHUNK_SIZE);
+
+        if self.decoded_len > 0 {
+            // we have a few leftover decoded bytes; flush that rather than pull in more b64
+            self.flush_decoded_buf(buf)
+        } else {
+            let mut at_eof = false;
+            while self.b64_len < BASE64_CHUNK_SIZE {
+                // Work around lack of copy_within, which is only present in 1.37
+                // Copy any bytes we have to the start of the buffer.
+                // We know we have < 1 chunk, so we can use a tiny tmp buffer.
+                let mut memmove_buf = [0_u8; BASE64_CHUNK_SIZE];
+                memmove_buf[..self.b64_len].copy_from_slice(
+                    &self.b64_buffer[self.b64_offset..self.b64_offset + self.b64_len],
+                );
+                self.b64_buffer[0..self.b64_len].copy_from_slice(&memmove_buf[..self.b64_len]);
+                self.b64_offset = 0;
+
+                // then fill in more data
+                let read = self.read_from_delegate()?;
+                if read == 0 {
+                    // we never pass in an empty buf, so 0 => we've hit EOF
+                    at_eof = true;
+                    break;
+                }
+            }
+
+            if self.b64_len == 0 {
+                debug_assert!(at_eof);
+                // we must be at EOF, and we have no data left to decode
+                return Ok(0);
+            };
+
+            debug_assert!(if at_eof {
+                // if we are at eof, we may not have a complete chunk
+                self.b64_len > 0
+            } else {
+                // otherwise, we must have at least one chunk
+                self.b64_len >= BASE64_CHUNK_SIZE
+            });
+
+            debug_assert_eq!(0, self.decoded_len);
+
+            if buf.len() < DECODED_CHUNK_SIZE {
+                // caller requested an annoyingly short read
+                // have to write to a tmp buf first to avoid double mutable borrow
+                let mut decoded_chunk = [0_u8; DECODED_CHUNK_SIZE];
+                // if we are at eof, could have less than BASE64_CHUNK_SIZE, in which case we have
+                // to assume that these last few tokens are, in fact, valid (i.e. must be 2-4 b64
+                // tokens, not 1, since 1 token can't decode to 1 byte).
+                let to_decode = cmp::min(self.b64_len, BASE64_CHUNK_SIZE);
+
+                let decoded = self.decode_to_buf(to_decode, &mut decoded_chunk[..])?;
+                self.decoded_buffer[..decoded].copy_from_slice(&decoded_chunk[..decoded]);
+
+                self.decoded_offset = 0;
+                self.decoded_len = decoded;
+
+                // can be less than 3 on last block due to padding
+                debug_assert!(decoded <= 3);
+
+                self.flush_decoded_buf(buf)
+            } else {
+                let b64_bytes_that_can_decode_into_buf = (buf.len() / DECODED_CHUNK_SIZE)
+                    .checked_mul(BASE64_CHUNK_SIZE)
+                    .expect("too many chunks");
+                debug_assert!(b64_bytes_that_can_decode_into_buf >= BASE64_CHUNK_SIZE);
+
+                let b64_bytes_available_to_decode = if at_eof {
+                    self.b64_len
+                } else {
+                    // only use complete chunks
+                    self.b64_len - self.b64_len % 4
+                };
+
+                let actual_decode_len = cmp::min(
+                    b64_bytes_that_can_decode_into_buf,
+                    b64_bytes_available_to_decode,
+                );
+                self.decode_to_buf(actual_decode_len, buf)
+            }
+        }
+    }
+}

zeroidc/vendor/base64/src/read/decoder_tests.rs

@@ -0,0 +1,335 @@
+use std::io::{self, Read};
+
+use rand::{Rng, RngCore};
+use std::{cmp, iter};
+
+use super::decoder::{DecoderReader, BUF_SIZE};
+use crate::encode::encode_config_buf;
+use crate::tests::random_config;
+use crate::{decode_config_buf, DecodeError, STANDARD};
+
+#[test]
+fn simple() {
+    let tests: &[(&[u8], &[u8])] = &[
+        (&b"0"[..], &b"MA=="[..]),
+        (b"01", b"MDE="),
+        (b"012", b"MDEy"),
+        (b"0123", b"MDEyMw=="),
+        (b"01234", b"MDEyMzQ="),
+        (b"012345", b"MDEyMzQ1"),
+        (b"0123456", b"MDEyMzQ1Ng=="),
+        (b"01234567", b"MDEyMzQ1Njc="),
+        (b"012345678", b"MDEyMzQ1Njc4"),
+        (b"0123456789", b"MDEyMzQ1Njc4OQ=="),
+    ][..];
+
+    for (text_expected, base64data) in tests.iter() {
+        // Read n bytes at a time.
+        for n in 1..base64data.len() + 1 {
+            let mut wrapped_reader = io::Cursor::new(base64data);
+            let mut decoder = DecoderReader::new(&mut wrapped_reader, STANDARD);
+
+            // handle errors as you normally would
+            let mut text_got = Vec::new();
+            let mut buffer = vec![0u8; n];
+            while let Ok(read) = decoder.read(&mut buffer[..]) {
+                if read == 0 {
+                    break;
+                }
+                text_got.extend_from_slice(&buffer[..read]);
+            }
+
+            assert_eq!(
+                text_got,
+                *text_expected,
+                "\nGot: {}\nExpected: {}",
+                String::from_utf8_lossy(&text_got[..]),
+                String::from_utf8_lossy(text_expected)
+            );
+        }
+    }
+}
+
+// Make sure we error out on trailing junk.
+#[test]
+fn trailing_junk() {
+    let tests: &[&[u8]] = &[&b"MDEyMzQ1Njc4*!@#$%^&"[..], b"MDEyMzQ1Njc4OQ== "][..];
+
+    for base64data in tests.iter() {
+        // Read n bytes at a time.
+        for n in 1..base64data.len() + 1 {
+            let mut wrapped_reader = io::Cursor::new(base64data);
+            let mut decoder = DecoderReader::new(&mut wrapped_reader, STANDARD);
+
+            // handle errors as you normally would
+            let mut buffer = vec![0u8; n];
+            let mut saw_error = false;
+            loop {
+                match decoder.read(&mut buffer[..]) {
+                    Err(_) => {
+                        saw_error = true;
+                        break;
+                    }
+                    Ok(read) if read == 0 => break,
+                    Ok(_) => (),
+                }
+            }
+
+            assert!(saw_error);
+        }
+    }
+}
+
+#[test]
+fn handles_short_read_from_delegate() {
+    let mut rng = rand::thread_rng();
+    let mut bytes = Vec::new();
+    let mut b64 = String::new();
+    let mut decoded = Vec::new();
+
+    for _ in 0..10_000 {
+        bytes.clear();
+        b64.clear();
+        decoded.clear();
+
+        let size = rng.gen_range(0, 10 * BUF_SIZE);
+        bytes.extend(iter::repeat(0).take(size));
+        bytes.truncate(size);
+        rng.fill_bytes(&mut bytes[..size]);
+        assert_eq!(size, bytes.len());
+
+        let config = random_config(&mut rng);
+        encode_config_buf(&bytes[..], config, &mut b64);
+
+        let mut wrapped_reader = io::Cursor::new(b64.as_bytes());
+        let mut short_reader = RandomShortRead {
+            delegate: &mut wrapped_reader,
+            rng: &mut rng,
+        };
+
+        let mut decoder = DecoderReader::new(&mut short_reader, config);
+
+        let decoded_len = decoder.read_to_end(&mut decoded).unwrap();
+        assert_eq!(size, decoded_len);
+        assert_eq!(&bytes[..], &decoded[..]);
+    }
+}
+
+#[test]
+fn read_in_short_increments() {
+    let mut rng = rand::thread_rng();
+    let mut bytes = Vec::new();
+    let mut b64 = String::new();
+    let mut decoded = Vec::new();
+
+    for _ in 0..10_000 {
+        bytes.clear();
+        b64.clear();
+        decoded.clear();
+
+        let size = rng.gen_range(0, 10 * BUF_SIZE);
+        bytes.extend(iter::repeat(0).take(size));
+        // leave room to play around with larger buffers
+        decoded.extend(iter::repeat(0).take(size * 3));
+
+        rng.fill_bytes(&mut bytes[..]);
+        assert_eq!(size, bytes.len());
+
+        let config = random_config(&mut rng);
+
+        encode_config_buf(&bytes[..], config, &mut b64);
+
+        let mut wrapped_reader = io::Cursor::new(&b64[..]);
+        let mut decoder = DecoderReader::new(&mut wrapped_reader, config);
+
+        consume_with_short_reads_and_validate(&mut rng, &bytes[..], &mut decoded, &mut decoder);
+    }
+}
+
+#[test]
+fn read_in_short_increments_with_short_delegate_reads() {
+    let mut rng = rand::thread_rng();
+    let mut bytes = Vec::new();
+    let mut b64 = String::new();
+    let mut decoded = Vec::new();
+
+    for _ in 0..10_000 {
+        bytes.clear();
+        b64.clear();
+        decoded.clear();
+
+        let size = rng.gen_range(0, 10 * BUF_SIZE);
+        bytes.extend(iter::repeat(0).take(size));
+        // leave room to play around with larger buffers
+        decoded.extend(iter::repeat(0).take(size * 3));
+
+        rng.fill_bytes(&mut bytes[..]);
+        assert_eq!(size, bytes.len());
+
+        let config = random_config(&mut rng);
+
+        encode_config_buf(&bytes[..], config, &mut b64);
+
+        let mut base_reader = io::Cursor::new(&b64[..]);
+        let mut decoder = DecoderReader::new(&mut base_reader, config);
+        let mut short_reader = RandomShortRead {
+            delegate: &mut decoder,
+            rng: &mut rand::thread_rng(),
+        };
+
+        consume_with_short_reads_and_validate(&mut rng, &bytes[..], &mut decoded, &mut short_reader)
+    }
+}
+
+#[test]
+fn reports_invalid_last_symbol_correctly() {
+    let mut rng = rand::thread_rng();
+    let mut bytes = Vec::new();
+    let mut b64 = String::new();
+    let mut b64_bytes = Vec::new();
+    let mut decoded = Vec::new();
+    let mut bulk_decoded = Vec::new();
+
+    for _ in 0..1_000 {
+        bytes.clear();
+        b64.clear();
+        b64_bytes.clear();
+
+        let size = rng.gen_range(1, 10 * BUF_SIZE);
+        bytes.extend(iter::repeat(0).take(size));
+        decoded.extend(iter::repeat(0).take(size));
+        rng.fill_bytes(&mut bytes[..]);
+        assert_eq!(size, bytes.len());
+
+        let mut config = random_config(&mut rng);
+        // changing padding will cause invalid padding errors when we twiddle the last byte
+        config.pad = false;
+
+        encode_config_buf(&bytes[..], config, &mut b64);
+        b64_bytes.extend(b64.bytes());
+        assert_eq!(b64_bytes.len(), b64.len());
+
+        // change the last character to every possible symbol. Should behave the same as bulk
+        // decoding whether invalid or valid.
+        for &s1 in config.char_set.encode_table().iter() {
+            decoded.clear();
+            bulk_decoded.clear();
+
+            // replace the last
+            *b64_bytes.last_mut().unwrap() = s1;
+            let bulk_res = decode_config_buf(&b64_bytes[..], config, &mut bulk_decoded);
+
+            let mut wrapped_reader = io::Cursor::new(&b64_bytes[..]);
+            let mut decoder = DecoderReader::new(&mut wrapped_reader, config);
+
+            let stream_res = decoder.read_to_end(&mut decoded).map(|_| ()).map_err(|e| {
+                e.into_inner()
+                    .and_then(|e| e.downcast::<DecodeError>().ok())
+            });
+
+            assert_eq!(bulk_res.map_err(|e| Some(Box::new(e))), stream_res);
+        }
+    }
+}
+
+#[test]
+fn reports_invalid_byte_correctly() {
+    let mut rng = rand::thread_rng();
+    let mut bytes = Vec::new();
+    let mut b64 = String::new();
+    let mut decoded = Vec::new();
+
+    for _ in 0..10_000 {
+        bytes.clear();
+        b64.clear();
+        decoded.clear();
+
+        let size = rng.gen_range(1, 10 * BUF_SIZE);
+        bytes.extend(iter::repeat(0).take(size));
+        rng.fill_bytes(&mut bytes[..size]);
+        assert_eq!(size, bytes.len());
+
+        let config = random_config(&mut rng);
+        encode_config_buf(&bytes[..], config, &mut b64);
+        // replace one byte, somewhere, with '*', which is invalid
+        let bad_byte_pos = rng.gen_range(0, &b64.len());
+        let mut b64_bytes = b64.bytes().collect::<Vec<u8>>();
+        b64_bytes[bad_byte_pos] = b'*';
+
+        let mut wrapped_reader = io::Cursor::new(b64_bytes.clone());
+        let mut decoder = DecoderReader::new(&mut wrapped_reader, config);
+
+        // some gymnastics to avoid double-moving the io::Error, which is not Copy
+        let read_decode_err = decoder
+            .read_to_end(&mut decoded)
+            .map_err(|e| {
+                let kind = e.kind();
+                let inner = e
+                    .into_inner()
+                    .and_then(|e| e.downcast::<DecodeError>().ok());
+                inner.map(|i| (*i, kind))
+            })
+            .err()
+            .and_then(|o| o);
+
+        let mut bulk_buf = Vec::new();
+        let bulk_decode_err = decode_config_buf(&b64_bytes[..], config, &mut bulk_buf).err();
+
+        // it's tricky to predict where the invalid data's offset will be since if it's in the last
+        // chunk it will be reported at the first padding location because it's treated as invalid
+        // padding. So, we just check that it's the same as it is for decoding all at once.
+        assert_eq!(
+            bulk_decode_err.map(|e| (e, io::ErrorKind::InvalidData)),
+            read_decode_err
+        );
+    }
+}
+
+fn consume_with_short_reads_and_validate<R: Read>(
+    rng: &mut rand::rngs::ThreadRng,
+    expected_bytes: &[u8],
+    decoded: &mut Vec<u8>,
+    short_reader: &mut R,
+) -> () {
+    let mut total_read = 0_usize;
+    loop {
+        assert!(
+            total_read <= expected_bytes.len(),
+            "tr {} size {}",
+            total_read,
+            expected_bytes.len()
+        );
+        if total_read == expected_bytes.len() {
+            assert_eq!(expected_bytes, &decoded[..total_read]);
+            // should be done
+            assert_eq!(0, short_reader.read(&mut decoded[..]).unwrap());
+            // didn't write anything
+            assert_eq!(expected_bytes, &decoded[..total_read]);
+
+            break;
+        }
+        let decode_len = rng.gen_range(1, cmp::max(2, expected_bytes.len() * 2));
+
+        let read = short_reader
+            .read(&mut decoded[total_read..total_read + decode_len])
+            .unwrap();
+        total_read += read;
+    }
+}
+
+/// Limits how many bytes a reader will provide in each read call.
+/// Useful for shaking out code that may work fine only with typical input sources that always fill
+/// the buffer.
+struct RandomShortRead<'a, 'b, R: io::Read, N: rand::Rng> {
+    delegate: &'b mut R,
+    rng: &'a mut N,
+}
+
+impl<'a, 'b, R: io::Read, N: rand::Rng> io::Read for RandomShortRead<'a, 'b, R, N> {
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> {
+        // avoid 0 since it means EOF for non-empty buffers
+        let effective_len = cmp::min(self.rng.gen_range(1, 20), buf.len());
+
+        self.delegate.read(&mut buf[..effective_len])
+    }
+}

zeroidc/vendor/base64/src/read/mod.rs

@@ -0,0 +1,6 @@
+//! Implementations of `io::Read` to transparently decode base64.
+mod decoder;
+pub use self::decoder::DecoderReader;
+
+#[cfg(test)]
+mod decoder_tests;