Merge pull request #157 from marshallpierce/mp/engine

Introduce the `Engine` abstraction.

.circleci/config.yml

@@ -12,7 +12,7 @@ workflows:
                   # be easier on the CI hosts since presumably those fat lower layers will already be cached, and
                   # therefore faster than a minimal, customized alpine.
                   # MSRV
-                  'rust:1.42.0',
+                  'rust:1.47.0',
                   # stable
                   'rust:latest',
                   'rustlang/rust:nightly'
@@ -67,8 +67,8 @@ jobs:
           command: |
             if [[ '<< parameters.rust_img >>' = 'rustlang/rust:nightly' ]]
             then
-              cargo +nightly install cargo-fuzz
-              cargo fuzz list | xargs -L 1 -I FUZZER cargo fuzz run FUZZER -- -max_total_time=1
+              cargo install cargo-fuzz
+              cargo fuzz list | xargs -I FUZZER cargo fuzz run FUZZER -- -max_total_time=1
             fi
 
       - save_cache:

.gitignore

@@ -10,5 +10,5 @@ main.rs
 *.iml
 
 # `perf record` files
-perf.data*
+/*perf.data*
 /tmp

Cargo.toml

@@ -19,6 +19,9 @@ harness = false
 criterion = "0.3.4"
 rand = "0.6.1"
 structopt = "0.3.21"
+# test fixtures for engine tests
+rstest = "0.11.0"
+rstest_reuse = "0.1.3"
 
 [features]
 default = ["std"]
@@ -28,3 +31,7 @@ std = []
 [profile.bench]
 # Useful for better disassembly when using `perf record` and `perf annotate`
 debug = true
+
+[profile.test]
+# Faster tests save much more than the increase in compilation time
+opt-level = 3

README.md

@@ -9,7 +9,7 @@ Made with CLion. Thanks to JetBrains for supporting open source!
 
 It's base64. What more could anyone want?
 
-This library's goals are to be *correct* and *fast*. It's thoroughly tested and widely used. It exposes functionality at multiple levels of abstraction so you can choose the level of convenience vs performance that you want, e.g. `decode_config_slice` decodes into an existing `&mut [u8]` and is pretty fast (2.6GiB/s for a 3 KiB input), whereas `decode_config` allocates a new `Vec<u8>` and returns it, which might be more convenient in some cases, but is slower (although still fast enough for almost any purpose) at 2.1 GiB/s.
+This library's goals are to be *correct* and *fast*. It's thoroughly tested and widely used. It exposes functionality at multiple levels of abstraction so you can choose the level of convenience vs performance that you want, e.g. `decode_engine_slice` decodes into an existing `&mut [u8]` and is pretty fast (2.6GiB/s for a 3 KiB input), whereas `decode_engine` allocates a new `Vec<u8>` and returns it, which might be more convenient in some cases, but is slower (although still fast enough for almost any purpose) at 2.1 GiB/s.
 
 Example
 ---
@@ -33,7 +33,7 @@ See the [docs](https://docs.rs/base64) for all the details.
 Rust version compatibility
 ---
 
-The minimum required Rust version is 1.36.0.
+The minimum required Rust version is 1.47.0.
 
 # Contributing
 
@@ -50,12 +50,6 @@ Benchmarks are in `benches/`. Running them requires nightly rust, but `rustup` m
 rustup run nightly cargo bench
 ```
 
-Decoding is aided by some pre-calculated tables, which are generated by:
-
-```bash
-cargo run --example make_tables > src/tables.rs.tmp && mv src/tables.rs.tmp src/tables.rs
-```
-
 no_std
 ---
 

RELEASE-NOTES.md

@@ -1,7 +1,13 @@
-# 0.14.0
-
-- MSRV is now 1.42.0
-- DecoderReader now owns its inner reader, and can expose it via `into_inner()`. For symmetry, `EncoderWriter` can do the same with its writer.
+# 0.20.0
+
+- Extended the `Config` concept into the `Engine` abstraction, allowing the user to pick different encoding / decoding implementations.
+  - What was formerly the only algorithm is now the `FastPortable` engine, so named because it's portable (works on any CPU) and relatively fast.
+  - This opens the door to a portable constant-time implementation ([#153](https://github.com/marshallpierce/rust-base64/pull/153), presumably `ConstantTimePortable`?) for security-sensitive applications that need side-channel resistance, and CPU-specific SIMD implementations for  more speed.
+  - Standard base64 per the RFC is available via `DEFAULT_ENGINE`. To use different alphabets or other settings (padding, etc), create your own engine instance.
+- `CharacterSet` is now `Alphabet` (per the RFC), and allows creating custom alphabets. The corresponding tables that were previously code-generated are now built dynamically.
+- Since there are already multiple breaking changes, various functions are renamed to be more consistent and discoverable.
+- MSRV is now 1.47.0 to allow various things to use `const fn`.
+- `DecoderReader` now owns its inner reader, and can expose it via `into_inner()`. For symmetry, `EncoderWriter` can do the same with its writer.
 
 # 0.13.0
 

benches/benchmarks.rs

@@ -5,16 +5,15 @@ extern crate rand;
 
 use base64::display;
 use base64::{
-    decode, decode_config_buf, decode_config_slice, encode, encode_config_buf, encode_config_slice,
-    write, Config,
+    decode, decode_engine_slice, decode_engine_vec, encode, encode_engine_slice,
+    encode_engine_string, write,
 };
 
+use base64::engine::DEFAULT_ENGINE;
 use criterion::{black_box, Bencher, BenchmarkId, Criterion, Throughput};
 use rand::{FromEntropy, Rng};
 use std::io::{self, Read, Write};
 
-const TEST_CONFIG: Config = base64::STANDARD;
-
 fn do_decode_bench(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4);
     fill(&mut v);
@@ -33,7 +32,7 @@ fn do_decode_bench_reuse_buf(b: &mut Bencher, &size: &usize) {
 
     let mut buf = Vec::new();
     b.iter(|| {
-        decode_config_buf(&encoded, TEST_CONFIG, &mut buf).unwrap();
+        decode_engine_vec(&encoded, &mut buf, &DEFAULT_ENGINE).unwrap();
         black_box(&buf);
         buf.clear();
     });
@@ -47,7 +46,7 @@ fn do_decode_bench_slice(b: &mut Bencher, &size: &usize) {
     let mut buf = Vec::new();
     buf.resize(size, 0);
     b.iter(|| {
-        decode_config_slice(&encoded, TEST_CONFIG, &mut buf).unwrap();
+        decode_engine_slice(&encoded, &mut buf, &DEFAULT_ENGINE).unwrap();
         black_box(&buf);
     });
 }
@@ -63,7 +62,7 @@ fn do_decode_bench_stream(b: &mut Bencher, &size: &usize) {
 
     b.iter(|| {
         let mut cursor = io::Cursor::new(&encoded[..]);
-        let mut decoder = base64::read::DecoderReader::new(&mut cursor, TEST_CONFIG);
+        let mut decoder = base64::read::DecoderReader::from(&mut cursor, &DEFAULT_ENGINE);
         decoder.read_to_end(&mut buf).unwrap();
         buf.clear();
         black_box(&buf);
@@ -83,7 +82,7 @@ fn do_encode_bench_display(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size);
     fill(&mut v);
     b.iter(|| {
-        let e = format!("{}", display::Base64Display::with_config(&v, TEST_CONFIG));
+        let e = format!("{}", display::Base64Display::from(&v, &DEFAULT_ENGINE));
         black_box(&e);
     });
 }
@@ -93,7 +92,7 @@ fn do_encode_bench_reuse_buf(b: &mut Bencher, &size: &usize) {
     fill(&mut v);
     let mut buf = String::new();
     b.iter(|| {
-        encode_config_buf(&v, TEST_CONFIG, &mut buf);
+        encode_engine_string(&v, &mut buf, &DEFAULT_ENGINE);
         buf.clear();
     });
 }
@@ -105,7 +104,7 @@ fn do_encode_bench_slice(b: &mut Bencher, &size: &usize) {
     // conservative estimate of encoded size
     buf.resize(v.len() * 2, 0);
     b.iter(|| {
-        encode_config_slice(&v, TEST_CONFIG, &mut buf);
+        encode_engine_slice(&v, &mut buf, &DEFAULT_ENGINE);
     });
 }
 
@@ -117,7 +116,7 @@ fn do_encode_bench_stream(b: &mut Bencher, &size: &usize) {
     buf.reserve(size * 2);
     b.iter(|| {
         buf.clear();
-        let mut stream_enc = write::EncoderWriter::new(&mut buf, TEST_CONFIG);
+        let mut stream_enc = write::EncoderWriter::from(&mut buf, &DEFAULT_ENGINE);
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
     });
@@ -128,7 +127,7 @@ fn do_encode_bench_string_stream(b: &mut Bencher, &size: &usize) {
     fill(&mut v);
 
     b.iter(|| {
-        let mut stream_enc = write::EncoderStringWriter::new(TEST_CONFIG);
+        let mut stream_enc = write::EncoderStringWriter::from(&DEFAULT_ENGINE);
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
         let _ = stream_enc.into_inner();
@@ -142,7 +141,7 @@ fn do_encode_bench_string_reuse_buf_stream(b: &mut Bencher, &size: &usize) {
     let mut buf = String::new();
     b.iter(|| {
         buf.clear();
-        let mut stream_enc = write::EncoderStringWriter::from(&mut buf, TEST_CONFIG);
+        let mut stream_enc = write::EncoderStringWriter::from_consumer(&mut buf, &DEFAULT_ENGINE);
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
         let _ = stream_enc.into_inner();

examples/base64.rs

@@ -4,37 +4,28 @@ use std::path::PathBuf;
 use std::process;
 use std::str::FromStr;
 
-use base64::{read, write};
+use base64::{alphabet, engine, read, write};
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
-enum CharacterSet {
+enum Alphabet {
     Standard,
     UrlSafe,
 }
 
-impl Default for CharacterSet {
+impl Default for Alphabet {
     fn default() -> Self {
-        CharacterSet::Standard
+        Alphabet::Standard
     }
 }
 
-impl Into<base64::Config> for CharacterSet {
-    fn into(self) -> base64::Config {
-        match self {
-            CharacterSet::Standard => base64::STANDARD,
-            CharacterSet::UrlSafe => base64::URL_SAFE,
-        }
-    }
-}
-
-impl FromStr for CharacterSet {
+impl FromStr for Alphabet {
     type Err = String;
-    fn from_str(s: &str) -> Result<CharacterSet, String> {
+    fn from_str(s: &str) -> Result<Alphabet, String> {
         match s {
-            "standard" => Ok(CharacterSet::Standard),
-            "urlsafe" => Ok(CharacterSet::UrlSafe),
-            _ => Err(format!("charset '{}' unrecognized", s)),
+            "standard" => Ok(Alphabet::Standard),
+            "urlsafe" => Ok(Alphabet::UrlSafe),
+            _ => Err(format!("alphabet '{}' unrecognized", s)),
         }
     }
 }
@@ -45,10 +36,10 @@ struct Opt {
     /// decode data
     #[structopt(short = "d", long = "decode")]
     decode: bool,
-    /// The character set to choose. Defaults to the standard base64 character set.
-    /// Supported character sets include "standard" and "urlsafe".
-    #[structopt(long = "charset")]
-    charset: Option<CharacterSet>,
+    /// The alphabet to choose. Defaults to the standard base64 alphabet.
+    /// Supported alphabets include "standard" and "urlsafe".
+    #[structopt(long = "alphabet")]
+    alphabet: Option<Alphabet>,
     /// The file to encode/decode.
     #[structopt(parse(from_os_str))]
     file: Option<PathBuf>,
@@ -68,14 +59,23 @@ fn main() {
         }
         Some(f) => Box::new(File::open(f).unwrap()),
     };
-    let config = opt.charset.unwrap_or_default().into();
+
+    let alphabet = opt.alphabet.unwrap_or_default();
+    let engine = engine::fast_portable::FastPortable::from(
+        &match alphabet {
+            Alphabet::Standard => alphabet::STANDARD,
+            Alphabet::UrlSafe => alphabet::URL_SAFE,
+        },
+        engine::fast_portable::PAD,
+    );
+
     let stdout = io::stdout();
     let mut stdout = stdout.lock();
     let r = if opt.decode {
-        let mut decoder = read::DecoderReader::new(&mut input, config);
+        let mut decoder = read::DecoderReader::from(&mut input, &engine);
         io::copy(&mut decoder, &mut stdout)
     } else {
-        let mut encoder = write::EncoderWriter::new(&mut stdout, config);
+        let mut encoder = write::EncoderWriter::from(&mut stdout, &engine);
         io::copy(&mut input, &mut encoder)
     };
     if let Err(e) = r {