From 6893986a029196ca42cd907597162050f344065c Mon Sep 17 00:00:00 2001 From: cyberdot Date: Sat, 19 Sep 2026 18:59:35 +0100 Subject: [PATCH 1/2] Base91 implementation --- CyberDot.Encoding.sln | 16 + .../CyberDot.Encoding.Base2048.Tests.csproj | 4 +- .../CyberDot.Encoding.Base65536.Tests.csproj | 4 +- .../CyberDot.Encoding.Base84.Tests.csproj | 14 +- .../Base91Tests.cs | 182 ++++++++++++ .../CyberDot.Encoding.Base91.Tests.csproj | 32 ++ .../StreamTransformTests.cs | 278 ++++++++++++++++++ src/CyberDot.Encoding.Base91/Base91.cs | 167 +++++++++++ src/CyberDot.Encoding.Base91/Constants.cs | 33 +++ .../CyberDot.Encoding.Base91.csproj | 24 ++ .../FromBase91Transform.cs | 106 +++++++ src/CyberDot.Encoding.Base91/README.md | 49 +++ .../ToBase91Transform.cs | 94 ++++++ 13 files changed, 995 insertions(+), 8 deletions(-) create mode 100644 src/CyberDot.Encoding.Base91.Tests/Base91Tests.cs create mode 100644 src/CyberDot.Encoding.Base91.Tests/CyberDot.Encoding.Base91.Tests.csproj create mode 100644 src/CyberDot.Encoding.Base91.Tests/StreamTransformTests.cs create mode 100644 src/CyberDot.Encoding.Base91/Base91.cs create mode 100644 src/CyberDot.Encoding.Base91/Constants.cs create mode 100644 src/CyberDot.Encoding.Base91/CyberDot.Encoding.Base91.csproj create mode 100644 src/CyberDot.Encoding.Base91/FromBase91Transform.cs create mode 100644 src/CyberDot.Encoding.Base91/README.md create mode 100644 src/CyberDot.Encoding.Base91/ToBase91Transform.cs diff --git a/CyberDot.Encoding.sln b/CyberDot.Encoding.sln index 5aa16dd..510d7cf 100644 --- a/CyberDot.Encoding.sln +++ b/CyberDot.Encoding.sln @@ -21,6 +21,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CyberDot.Encoding.Base84.Te EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Base84", "Base84", "{4C6BEADC-6B8C-425C-9EC4-559597A7915C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CyberDot.Encoding.Base91", "src\CyberDot.Encoding.Base91\CyberDot.Encoding.Base91.csproj", "{7454C92C-8908-4EE6-8A87-FED28AA7F6BE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CyberDot.Encoding.Base91.Tests", "src\CyberDot.Encoding.Base91.Tests\CyberDot.Encoding.Base91.Tests.csproj", "{D14013B6-8905-46BF-8B40-61AE19476DF0}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Base91", "Base91", "{6BF31631-01AB-4D7B-A0D0-512C70365327}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -51,6 +57,14 @@ Global {3762FA86-26D7-4037-B7CF-E5CA94F03D8B}.Debug|Any CPU.Build.0 = Debug|Any CPU {3762FA86-26D7-4037-B7CF-E5CA94F03D8B}.Release|Any CPU.ActiveCfg = Release|Any CPU {3762FA86-26D7-4037-B7CF-E5CA94F03D8B}.Release|Any CPU.Build.0 = Release|Any CPU + {7454C92C-8908-4EE6-8A87-FED28AA7F6BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7454C92C-8908-4EE6-8A87-FED28AA7F6BE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7454C92C-8908-4EE6-8A87-FED28AA7F6BE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7454C92C-8908-4EE6-8A87-FED28AA7F6BE}.Release|Any CPU.Build.0 = Release|Any CPU + {D14013B6-8905-46BF-8B40-61AE19476DF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D14013B6-8905-46BF-8B40-61AE19476DF0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D14013B6-8905-46BF-8B40-61AE19476DF0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D14013B6-8905-46BF-8B40-61AE19476DF0}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -62,6 +76,8 @@ Global {287EE26D-CAE7-4AA3-93C5-4A6B2C8A9D05} = {7F2B409E-7822-462C-8152-0DE5BEE42CAD} {9551EC7C-8F4A-462A-AABC-50D1FD1E60B5} = {4C6BEADC-6B8C-425C-9EC4-559597A7915C} {3762FA86-26D7-4037-B7CF-E5CA94F03D8B} = {4C6BEADC-6B8C-425C-9EC4-559597A7915C} + {7454C92C-8908-4EE6-8A87-FED28AA7F6BE} = {6BF31631-01AB-4D7B-A0D0-512C70365327} + {D14013B6-8905-46BF-8B40-61AE19476DF0} = {6BF31631-01AB-4D7B-A0D0-512C70365327} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9D040A49-47AE-4503-B1C2-7D3E82AF66AB} diff --git a/src/CyberDot.Encoding.Base2048.Tests/CyberDot.Encoding.Base2048.Tests.csproj b/src/CyberDot.Encoding.Base2048.Tests/CyberDot.Encoding.Base2048.Tests.csproj index f02f961..bc04c90 100644 --- a/src/CyberDot.Encoding.Base2048.Tests/CyberDot.Encoding.Base2048.Tests.csproj +++ b/src/CyberDot.Encoding.Base2048.Tests/CyberDot.Encoding.Base2048.Tests.csproj @@ -8,8 +8,8 @@ - - + + all diff --git a/src/CyberDot.Encoding.Base65536.Tests/CyberDot.Encoding.Base65536.Tests.csproj b/src/CyberDot.Encoding.Base65536.Tests/CyberDot.Encoding.Base65536.Tests.csproj index 19f232d..9f5d157 100644 --- a/src/CyberDot.Encoding.Base65536.Tests/CyberDot.Encoding.Base65536.Tests.csproj +++ b/src/CyberDot.Encoding.Base65536.Tests/CyberDot.Encoding.Base65536.Tests.csproj @@ -5,8 +5,8 @@ - - + + all diff --git a/src/CyberDot.Encoding.Base84.Tests/CyberDot.Encoding.Base84.Tests.csproj b/src/CyberDot.Encoding.Base84.Tests/CyberDot.Encoding.Base84.Tests.csproj index 2b5fa8a..2bb740c 100644 --- a/src/CyberDot.Encoding.Base84.Tests/CyberDot.Encoding.Base84.Tests.csproj +++ b/src/CyberDot.Encoding.Base84.Tests/CyberDot.Encoding.Base84.Tests.csproj @@ -8,11 +8,17 @@ - - - + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/CyberDot.Encoding.Base91.Tests/Base91Tests.cs b/src/CyberDot.Encoding.Base91.Tests/Base91Tests.cs new file mode 100644 index 0000000..e2b80bb --- /dev/null +++ b/src/CyberDot.Encoding.Base91.Tests/Base91Tests.cs @@ -0,0 +1,182 @@ +using AwesomeAssertions; + +namespace CyberDot.Encoding.Base91.Tests +{ + public class Base91Tests + { + // Vectors below were generated from a direct Python port of the reference basE91 + // algorithm (refimplementation.txt); "test" -> "fPNKd" also matches the canonical + // basE91 example from Joachim Henke's original implementation. + [Fact] + public void Should_encode_and_decode_known_vectors() + { + AssertVector("", ""); + AssertVector("M", "=A"); + AssertVector("Ma", "8DD"); + AssertVector("Man", "8D$J"); + AssertVector("Man ", "8D$JI"); + AssertVector("test", "fPNKd"); + AssertVector("Hello, World!", ">OwJh>}AQ;r@@Y?F"); + AssertVector("The quick brown fox jumps over the lazy dog.", + "nX^Iz?T1s!2t:aRn#o>vf>6C9#`##mlLK#_1:Wzv;RG!,a%q3Lc=5gA"); + } + + [Fact] + public void Should_encode_and_decode_runs_of_zero_bytes() + { + AssertVector(new byte[] { 0x00 }, "AA"); + AssertVector(new byte[] { 0x00, 0x00 }, "AAA"); + AssertVector(new byte[] { 0x00, 0x00, 0x00 }, "AAAA"); + AssertVector(new byte[] { 0x00, 0x00, 0x00, 0x00 }, "AAAAA"); + AssertVector(new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00 }, "AAAAAA"); + } + + [Fact] + public void Should_encode_and_decode_runs_of_ff_bytes() + { + AssertVector(new byte[] { 0xff }, "/C"); + AssertVector(new byte[] { 0xff, 0xff }, "B\"H"); + AssertVector(new byte[] { 0xff, 0xff, 0xff }, "B\"tW"); + AssertVector(new byte[] { 0xff, 0xff, 0xff, 0xff }, "B\"B\"#"); + AssertVector(new byte[] { 0xff, 0xff, 0xff, 0xff, 0xff }, "B\"B\"B\"B"); + } + + [Fact] + public void Should_encode_and_decode_mixed_bytes() + { + AssertVector(new byte[] { 0x00, 0xe0, 0xff, 0x01 }, "C\"tWA"); + AssertVector( + new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + ":C#(:C?hVB$MSiVEwndB"); + } + + [Theory] + [InlineData("AA ")] + [InlineData("AA'")] + [InlineData("AA-")] + [InlineData("AA\\")] + [InlineData("AA\x00")] + [InlineData("AA\x7f")] + public void Should_throw_on_unrecognised_character(string input) + { + var act = () => Base91.Decode(input); + + act.Should().Throw(); + } + + [Fact] + public void Should_throw_on_null_input() + { + var encodeAct = () => Base91.Encode(null); + var decodeAct = () => Base91.Decode(null); + + encodeAct.Should().Throw(); + decodeAct.Should().Throw(); + } + + [Fact] + public void Should_round_trip_all_single_bytes() + { + for (var b = 0; b <= 255; b++) + { + var data = new[] { (byte)b }; + + var encoded = Base91.Encode(data); + Base91.Decode(encoded).Should().BeEquivalentTo(data); + } + } + + [Fact] + public void Should_round_trip_all_byte_pairs() + { + for (var a = 0; a <= 255; a++) + { + for (var b = 0; b <= 255; b++) + { + var data = new[] { (byte)a, (byte)b }; + + var encoded = Base91.Encode(data); + Base91.Decode(encoded).Should().BeEquivalentTo(data); + } + } + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(17)] + [InlineData(31)] + [InlineData(32)] + [InlineData(33)] + [InlineData(256)] + public void Should_round_trip_random_data(int length) + { + var data = new byte[length]; + new Random(length).NextBytes(data); + + var encoded = Base91.Encode(data); + Base91.Decode(encoded).Should().BeEquivalentTo(data); + } + + [Fact] + public void Should_round_trip_many_random_lengths() + { + var random = new Random(12345); + var buffer = new byte[256]; + + for (var i = 0; i < 5000; i++) + { + var len = random.Next(0, buffer.Length + 1); + random.NextBytes(buffer); + var data = new byte[len]; + Array.Copy(buffer, data, len); + + var encoded = Base91.Encode(data); + Base91.Decode(encoded).Should().BeEquivalentTo(data); + } + } + + // Unlike Base84/Base2048, Base91 has no fixed-width character group, so decoding + // cannot validate padding - any string built purely from alphabet characters decodes + // without throwing, even if it was never produced by Encode. + [Fact] + public void Should_never_throw_when_decoding_any_string_of_alphabet_characters() + { + var random = new Random(999); + var buffer = new char[64]; + + for (var i = 0; i < 2000; i++) + { + var len = random.Next(0, buffer.Length + 1); + for (var j = 0; j < len; j++) + { + buffer[j] = AlphabetChars[random.Next(AlphabetChars.Length)]; + } + + var candidate = new string(buffer, 0, len); + + var act = () => Base91.Decode(candidate); + + act.Should().NotThrow(); + } + } + + private const string AlphabetChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&()*+,./:;<=>?@[]^_`{|}~\""; + + private static void AssertVector(string decoded, string encoded) + { + AssertVector(System.Text.Encoding.UTF8.GetBytes(decoded), encoded); + } + + private static void AssertVector(byte[] decoded, string encoded) + { + Base91.Encode(decoded).Should().Be(encoded); + Base91.Decode(encoded).Should().BeEquivalentTo(decoded); + } + } +} diff --git a/src/CyberDot.Encoding.Base91.Tests/CyberDot.Encoding.Base91.Tests.csproj b/src/CyberDot.Encoding.Base91.Tests/CyberDot.Encoding.Base91.Tests.csproj new file mode 100644 index 0000000..3a08e63 --- /dev/null +++ b/src/CyberDot.Encoding.Base91.Tests/CyberDot.Encoding.Base91.Tests.csproj @@ -0,0 +1,32 @@ + + + + net10.0 + enable + enable + false + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + \ No newline at end of file diff --git a/src/CyberDot.Encoding.Base91.Tests/StreamTransformTests.cs b/src/CyberDot.Encoding.Base91.Tests/StreamTransformTests.cs new file mode 100644 index 0000000..646ba20 --- /dev/null +++ b/src/CyberDot.Encoding.Base91.Tests/StreamTransformTests.cs @@ -0,0 +1,278 @@ +using System.Security.Cryptography; +using AwesomeAssertions; +using TextEncoding = System.Text.Encoding; + +namespace CyberDot.Encoding.Base91.Tests +{ + public class StreamTransformTests + { + private static readonly int[] ChunkSizes = [1, 2, 3, 4, 5, 7, 13, 64]; + + // Base91's alphabet is pure ASCII, so the encoded text is equally valid under any + // of these three encodings - the transforms support all of them. + private static readonly TextEncoding[] SafeEncodings = [TextEncoding.UTF8, TextEncoding.Unicode, TextEncoding.UTF32]; + + private static readonly (byte[] Decoded, string Encoded)[] Vectors = + { + (System.Array.Empty(), ""), + (TextEncoding.UTF8.GetBytes("M"), "=A"), + (TextEncoding.UTF8.GetBytes("Ma"), "8DD"), + (TextEncoding.UTF8.GetBytes("Man"), "8D$J"), + (TextEncoding.UTF8.GetBytes("Man "), "8D$JI"), + (TextEncoding.UTF8.GetBytes("test"), "fPNKd"), + (TextEncoding.UTF8.GetBytes("Hello, World!"), ">OwJh>}AQ;r@@Y?F"), + (new byte[] { 0xff, 0xff, 0xff, 0xff, 0xff }, "B\"B\"B\"B"), + ( + TextEncoding.UTF8.GetBytes("The quick brown fox jumps over the lazy dog."), + "nX^Iz?T1s!2t:aRn#o>vf>6C9#`##mlLK#_1:Wzv;RG!,a%q3Lc=5gA" + ), + }; + + private static byte[] EncodeViaCryptoStream(byte[] data, TextEncoding? encoding = null) + { + using var ms = new MemoryStream(); + using (var cs = new CryptoStream(ms, new ToBase91Transform(encoding), CryptoStreamMode.Write, leaveOpen: true)) + { + cs.Write(data, 0, data.Length); + cs.FlushFinalBlock(); + } + + return ms.ToArray(); + } + + private static byte[] DecodeViaCryptoStream(byte[] bytes, TextEncoding? encoding = null) + { + using var ms = new MemoryStream(bytes); + using var cs = new CryptoStream(ms, new FromBase91Transform(encoding), CryptoStreamMode.Read); + using var outMs = new MemoryStream(); + cs.CopyTo(outMs); + return outMs.ToArray(); + } + + private static byte[] DriveTransform(ICryptoTransform transform, byte[] input, int chunkSize) + { + using var outMs = new MemoryStream(); + var i = 0; + + while (input.Length - i > chunkSize) + { + var buffer = new byte[chunkSize * 8 + 32]; + var written = transform.TransformBlock(input, i, chunkSize, buffer, 0); + outMs.Write(buffer, 0, written); + i += chunkSize; + } + + var final = transform.TransformFinalBlock(input, i, input.Length - i); + outMs.Write(final, 0, final.Length); + + return outMs.ToArray(); + } + + private static byte[] AsBytes(string value, TextEncoding? encoding = null) => (encoding ?? TextEncoding.UTF8).GetBytes(value); + + // --- Round trip against the known vectors (default: UTF-8) ----------------------------------------------- + + [Fact] + public void Should_stream_encode_known_vectors() + { + foreach (var (decoded, encoded) in Vectors) + { + EncodeViaCryptoStream(decoded).Should().BeEquivalentTo(AsBytes(encoded)); + } + } + + [Fact] + public void Should_stream_decode_known_vectors() + { + foreach (var (decoded, encoded) in Vectors) + { + DecodeViaCryptoStream(AsBytes(encoded)).Should().BeEquivalentTo(decoded); + } + } + + [Theory] + [InlineData("AA ")] + [InlineData("AA'")] + [InlineData("AA\\")] + public void Should_throw_on_unrecognised_character_while_streaming(string input) + { + var act = () => DecodeViaCryptoStream(AsBytes(input)); + + act.Should().Throw(); + } + + // --- Round trip across all three "safe" encodings --------------------------------------------------------- + + [Fact] + public void Should_round_trip_known_vectors_across_all_safe_encodings() + { + foreach (var encoding in SafeEncodings) + { + foreach (var (decoded, encoded) in Vectors) + { + EncodeViaCryptoStream(decoded, encoding) + .Should().BeEquivalentTo(AsBytes(encoded, encoding), $"encoding {encoding.EncodingName}"); + + DecodeViaCryptoStream(AsBytes(encoded, encoding), encoding) + .Should().BeEquivalentTo(decoded, $"encoding {encoding.EncodingName}"); + } + } + } + + // --- Chunked feeding: exercises bit-queue buffering across calls ------------------------------------------- + + [Fact] + public void Should_encode_correctly_regardless_of_chunk_size() + { + foreach (var (decoded, encoded) in Vectors) + { + foreach (var encoding in SafeEncodings) + { + var expectedBytes = AsBytes(encoded, encoding); + + foreach (var chunkSize in ChunkSizes) + { + DriveTransform(new ToBase91Transform(encoding), decoded, chunkSize) + .Should().BeEquivalentTo(expectedBytes, $"chunk size {chunkSize}, encoding {encoding.EncodingName}"); + } + } + } + } + + [Fact] + public void Should_decode_correctly_regardless_of_chunk_size() + { + foreach (var (decoded, encoded) in Vectors) + { + foreach (var encoding in SafeEncodings) + { + var inputBytes = AsBytes(encoded, encoding); + + foreach (var chunkSize in ChunkSizes) + { + DriveTransform(new FromBase91Transform(encoding), inputBytes, chunkSize) + .Should().BeEquivalentTo(decoded, $"chunk size {chunkSize}, encoding {encoding.EncodingName}"); + } + } + } + } + + // --- Round trip fuzzing over random data of varying lengths ------------------------------------------------ + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(17)] + [InlineData(256)] + [InlineData(1000)] + [InlineData(4097)] + public void Should_round_trip_random_data_through_crypto_stream(int length) + { + var data = new byte[length]; + new Random(length).NextBytes(data); + + foreach (var encoding in SafeEncodings) + { + var encoded = EncodeViaCryptoStream(data, encoding); + encoded.Should().BeEquivalentTo(AsBytes(Base91.Encode(data), encoding), $"encoding {encoding.EncodingName}"); + + var decoded = DecodeViaCryptoStream(encoded, encoding); + decoded.Should().BeEquivalentTo(data, $"encoding {encoding.EncodingName}"); + } + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(17)] + [InlineData(256)] + public void Should_round_trip_random_data_with_odd_chunk_sizes(int length) + { + var data = new byte[length]; + new Random(length + 1).NextBytes(data); + + foreach (var encoding in SafeEncodings) + { + foreach (var chunkSize in ChunkSizes) + { + var encoded = DriveTransform(new ToBase91Transform(encoding), data, chunkSize); + var decoded = DriveTransform(new FromBase91Transform(encoding), encoded, chunkSize); + + decoded.Should().BeEquivalentTo(data, $"chunk size {chunkSize}, encoding {encoding.EncodingName}, length {length}"); + } + } + } + + // --- Transform metadata -------------------------------------------------------------------------------- + + [Theory] + [MemberData(nameof(SafeEncodingsMemberData))] + public void Should_expose_expected_metadata_on_encode(TextEncoding encoding) + { + var transform = new ToBase91Transform(encoding); + + transform.InputBlockSize.Should().Be(1); + transform.OutputBlockSize.Should().BeGreaterThanOrEqualTo(2); + transform.CanTransformMultipleBlocks.Should().BeTrue(); + transform.CanReuseTransform.Should().BeTrue(); + } + + [Theory] + [MemberData(nameof(SafeEncodingsMemberData))] + public void Should_expose_expected_metadata_on_decode(TextEncoding encoding) + { + var transform = new FromBase91Transform(encoding); + + transform.InputBlockSize.Should().Be(1); + transform.OutputBlockSize.Should().Be(1); + transform.CanTransformMultipleBlocks.Should().BeTrue(); + transform.CanReuseTransform.Should().BeTrue(); + } + + [Fact] + public void Should_default_to_utf8_when_no_encoding_supplied() + { + var data = new byte[] { 1, 2, 3, 4, 5 }; + + EncodeViaCryptoStream(data).Should().BeEquivalentTo(AsBytes(Base91.Encode(data), TextEncoding.UTF8)); + } + + public static TheoryData SafeEncodingsMemberData() + { + var data = new TheoryData(); + foreach (var encoding in SafeEncodings) data.Add(encoding); + return data; + } + + [Fact] + public void Should_reuse_transform_instances_across_multiple_streams() + { + var toTransform = new ToBase91Transform(); + var fromTransform = new FromBase91Transform(); + + var first = new byte[] { 1, 2, 3 }; + var second = new byte[] { 4, 5, 6, 7 }; + + foreach (var data in new[] { first, second }) + { + using var ms = new MemoryStream(); + using (var cs = new CryptoStream(ms, toTransform, CryptoStreamMode.Write, leaveOpen: true)) + { + cs.Write(data, 0, data.Length); + cs.FlushFinalBlock(); + } + + using var decodedMs = new MemoryStream(ms.ToArray()); + using var decodeCs = new CryptoStream(decodedMs, fromTransform, CryptoStreamMode.Read); + using var outMs = new MemoryStream(); + decodeCs.CopyTo(outMs); + + outMs.ToArray().Should().BeEquivalentTo(data); + } + } + } +} diff --git a/src/CyberDot.Encoding.Base91/Base91.cs b/src/CyberDot.Encoding.Base91/Base91.cs new file mode 100644 index 0000000..78a33a3 --- /dev/null +++ b/src/CyberDot.Encoding.Base91/Base91.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.Text; +using static CyberDot.Encoding.Base91.Constants; + +namespace CyberDot.Encoding.Base91 +{ + /// + /// Base91 (basE91) is a binary-to-text encoding that uses 91 of the 94 printable ASCII + /// characters - excluding space, apostrophe, hyphen and backslash, which are left free so + /// encoded text stays easy to quote or embed. It packs input bits into a queue and drains + /// a pair of characters at a time, each pair holding 13 or 14 input bits, so encoded + /// output is about 23% larger than the input, against 33% for Base64. + /// + /// Unlike Base64/Base85-style encodings, Base91 has no fixed-width character group, so + /// there is no canonical padding to validate on decode: bits left over from a final, + /// unpaired character are simply dropped, exactly as the reference implementation does. + /// + public static class Base91 + { + private const string AlphabetChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&()*+,./:;<=>?@[]^_`{|}~\""; + private const byte None = 0xff; + + internal static readonly char[] Alphabet = AlphabetChars.ToCharArray(); + internal static readonly byte[] InverseMap = BuildInverseMap(); + + private static byte[] BuildInverseMap() + { + var map = new byte[256]; + for (var i = 0; i < map.Length; i++) + { + map[i] = None; + } + + for (var i = 0; i < Alphabet.Length; i++) + { + map[Alphabet[i]] = (byte)i; + } + + return map; + } + + public static string Encode(byte[] data) + { + if (data == null) + { + throw new ArgumentNullException(nameof(data)); + } + + var result = new StringBuilder(data.Length + data.Length / 4 + 2); + uint queue = 0; + var numBits = 0; + + foreach (var b in data) + { + queue |= (uint)b << numBits; + numBits += BitsPerByte; + + if (numBits > QueueFlushThreshold) + { + var bits = NextPairBits(queue, out var v); + result.Append(Alphabet[v % Radix]); + result.Append(Alphabet[v / Radix]); + queue >>= bits; + numBits -= bits; + } + } + + if (numBits > 0) + { + result.Append(Alphabet[queue % Radix]); + if (!FitsOneChar(numBits, queue)) + { + result.Append(Alphabet[queue / Radix]); + } + } + + return result.ToString(); + } + + /// + /// Decodes a Base91 encoded string back into a byte slice. Since Base91 has no + /// canonical padding, this only rejects characters outside the Base91 alphabet - any + /// other input decodes, though data that was never validly encoded may not round trip. + /// + public static byte[] Decode(string data) + { + if (data == null) + { + throw new ArgumentNullException(nameof(data)); + } + + var output = new List(data.Length * 13 / 16 + 2); + uint queue = 0; + var numBits = 0; + var value = -1; + + foreach (var c in data) + { + var digit = Digit(c); + + if (value < 0) + { + value = digit; + continue; + } + + value += digit * Radix; + queue |= (uint)value << numBits; + numBits += PairBitsOf((uint)value); + + do + { + output.Add((byte)queue); + queue >>= BitsPerByte; + numBits -= BitsPerByte; + } while (numBits > SingleCharBitLimit); + + value = -1; + } + + if (value >= 0) + { + output.Add((byte)(queue | (uint)(value << numBits))); + } + + return output.ToArray(); + } + + // The number of input bits held by the next output pair, and the value to split + // across its two characters, given the current state of the bit queue. + internal static int NextPairBits(uint queue, out uint value) + { + value = queue & LowBitsMask; + if (value > SplitValue) + { + return QueueFlushThreshold; + } + + value = queue & WideBitsMask; + return QueueFlushThreshold + 1; + } + + // The number of input bits a just-combined character pair represents. + internal static int PairBitsOf(uint value) + { + return (value & LowBitsMask) > SplitValue ? QueueFlushThreshold : QueueFlushThreshold + 1; + } + + // Whether the encoder writes a single trailing character for this final queue state. + internal static bool FitsOneChar(int numBits, uint queue) + { + return numBits <= SingleCharBitLimit && queue <= SplitValue; + } + + internal static int Digit(char c) + { + var value = c < InverseMap.Length ? InverseMap[c] : None; + if (value == None) + { + throw new ArgumentException("Unrecognised Base91 character: '" + c + "'."); + } + + return value; + } + } +} diff --git a/src/CyberDot.Encoding.Base91/Constants.cs b/src/CyberDot.Encoding.Base91/Constants.cs new file mode 100644 index 0000000..fc68c7d --- /dev/null +++ b/src/CyberDot.Encoding.Base91/Constants.cs @@ -0,0 +1,33 @@ +namespace CyberDot.Encoding.Base91 +{ + // Numeric constants ported from the reference basE91 algorithm (refimplementation.txt). + // Base91 has no fixed-width character group like Base64/Base85: it packs input bits into + // a queue and drains a pair of characters at a time, each pair holding 13 or 14 bits + // depending on the combined value. Because of that, decoding cannot validate padding the + // way Base84/Base2048 do - any leftover bits in a final, unpaired character are simply + // dropped, exactly as the reference implementation does. + internal static class Constants + { + public const int BitsPerByte = 8; + public const int Radix = 91; + + // Once the bit queue holds more than this many bits, a pair of characters is drained. + public const int QueueFlushThreshold = 13; + + // Mask covering the low QueueFlushThreshold bits of the queue. + public const uint LowBitsMask = (1u << QueueFlushThreshold) - 1; + + // Mask covering one more bit than LowBitsMask, used when the low-bits value alone is + // too small to have come from a QueueFlushThreshold + 1 bit chunk. + public const uint WideBitsMask = (1u << (QueueFlushThreshold + 1)) - 1; + + // A low-bits value at or below this could only have come from a wider (14-bit) chunk, + // so the encoder/decoder fall back to WideBitsMask instead of LowBitsMask. + public const int SplitValue = 88; + + // When flushing a final partial queue, a single trailing character suffices only when + // strictly fewer than this many bits remain (and the queue's value is small enough - + // see SplitValue - to round-trip through one character alone). + public const int SingleCharBitLimit = 7; + } +} diff --git a/src/CyberDot.Encoding.Base91/CyberDot.Encoding.Base91.csproj b/src/CyberDot.Encoding.Base91/CyberDot.Encoding.Base91.csproj new file mode 100644 index 0000000..0d833e6 --- /dev/null +++ b/src/CyberDot.Encoding.Base91/CyberDot.Encoding.Base91.csproj @@ -0,0 +1,24 @@ + + + + + netstandard1.6 + True + 1.0.0 + CyberDot + CyberDot + An implementation of Base91 binary-to-text encoding in C#, including streaming encode/decode functionality via ICryptoTransform/CryptoStream. + Base91, Encoding + MIT + README.md + https://github.com/cyberdot/CyberDot.Encoding + https://github.com/cyberdot/CyberDot.Encoding + git + true + snupkg + + + + + + diff --git a/src/CyberDot.Encoding.Base91/FromBase91Transform.cs b/src/CyberDot.Encoding.Base91/FromBase91Transform.cs new file mode 100644 index 0000000..ad8d75b --- /dev/null +++ b/src/CyberDot.Encoding.Base91/FromBase91Transform.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using static CyberDot.Encoding.Base91.Constants; + +namespace CyberDot.Encoding.Base91 +{ + /// + /// Streams a Base91 sequence back into the original raw bytes. Since the Base91 alphabet + /// is pure ASCII, the input text may validly have been serialised with any + /// ; pass the matching one to the constructor (UTF-8 by default). + /// Intended for use with . + /// + public class FromBase91Transform : ICryptoTransform + { + private readonly Decoder decoder; + + // Bit accumulator and dangling-character state, carried across TransformBlock calls. + private uint queue; + private int numBits; + private int value = -1; + + public FromBase91Transform(System.Text.Encoding encoding = null) + { + decoder = (encoding ?? new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true)).GetDecoder(); + } + + public bool CanReuseTransform => true; + public bool CanTransformMultipleBlocks => true; + public int InputBlockSize => 1; + public int OutputBlockSize => 1; + + public void Dispose() { } + + public int TransformBlock(byte[] inputBuffer, int inputOffset, int inputCount, byte[] outputBuffer, int outputOffset) + { + var output = ProcessBytes(inputBuffer, inputOffset, inputCount, isFinal: false); + Array.Copy(output, 0, outputBuffer, outputOffset, output.Length); + return output.Length; + } + + public byte[] TransformFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount) + { + var result = ProcessBytes(inputBuffer, inputOffset, inputCount, isFinal: true); + Reset(); + return result; + } + + private void Reset() + { + decoder.Reset(); + queue = 0; + numBits = 0; + value = -1; + } + + // The Decoder buffers any incomplete multibyte sequence internally across calls (and, + // with a throwing Encoding, rejects malformed/truncated input), so we only need to + // track our own bit-accumulator state here. + private byte[] ProcessBytes(byte[] inputBuffer, int inputOffset, int inputCount, bool isFinal) + { + if (inputBuffer == null) + { + throw new ArgumentNullException(nameof(inputBuffer)); + } + + var maxChars = decoder.GetCharCount(inputBuffer, inputOffset, inputCount, isFinal); + var chars = maxChars == 0 ? Array.Empty() : new char[maxChars]; + var charCount = decoder.GetChars(inputBuffer, inputOffset, inputCount, chars, 0, isFinal); + + var output = new List(charCount); + + for (var i = 0; i < charCount; i++) + { + var digit = Base91.Digit(chars[i]); + + if (value < 0) + { + value = digit; + continue; + } + + value += digit * Radix; + queue |= (uint)value << numBits; + numBits += Base91.PairBitsOf((uint)value); + + do + { + output.Add((byte)queue); + queue >>= BitsPerByte; + numBits -= BitsPerByte; + } while (numBits > SingleCharBitLimit); + + value = -1; + } + + if (isFinal && value >= 0) + { + output.Add((byte)(queue | (uint)(value << numBits))); + } + + return output.ToArray(); + } + } +} diff --git a/src/CyberDot.Encoding.Base91/README.md b/src/CyberDot.Encoding.Base91/README.md new file mode 100644 index 0000000..04c3338 --- /dev/null +++ b/src/CyberDot.Encoding.Base91/README.md @@ -0,0 +1,49 @@ +# CyberDot.Encoding.Base91 + +An implementation of Base91 (basE91) encoding in C#. + +Base91 is a binary-to-text encoding that uses 91 of the 94 printable ASCII characters - +excluding space, apostrophe, hyphen and backslash. It packs input bits into a queue and +drains a pair of characters at a time, each pair holding 13 or 14 bits, so encoded output +is about 23% larger than the input, against 33% for Base64. + +Unlike Base64/Base85-style encodings, Base91 has no fixed-width character group, so there +is no canonical padding to validate on decode: bits left over from a final, unpaired +character are simply dropped, exactly as the reference implementation does. + +## Usage + +```csharp +using CyberDot.Encoding.Base91; + +var bytes = System.Text.Encoding.UTF8.GetBytes("hello world"); +var encoded = Base91.Encode(bytes); + +var decoded = Base91.Decode(encoded); // Output: hello world +``` + +### Streaming + +`ToBase91Transform`/`FromBase91Transform` implement `ICryptoTransform`, so they plug +into `CryptoStream` for streaming encode/decode without buffering the whole payload in +memory. They default to UTF-8, or accept any `Encoding` via the constructor. + +```csharp +using System.Security.Cryptography; + +using var output = new MemoryStream(); +using (var cryptoStream = new CryptoStream(output, new ToBase91Transform(), CryptoStreamMode.Write)) +{ + sourceStream.CopyTo(cryptoStream); +} + +using var decoded = new MemoryStream(); +using (var cryptoStream = new CryptoStream(encodedStream, new FromBase91Transform(), CryptoStreamMode.Read)) +{ + cryptoStream.CopyTo(decoded); +} +``` + +## License + +The MIT License (MIT) diff --git a/src/CyberDot.Encoding.Base91/ToBase91Transform.cs b/src/CyberDot.Encoding.Base91/ToBase91Transform.cs new file mode 100644 index 0000000..6c1085d --- /dev/null +++ b/src/CyberDot.Encoding.Base91/ToBase91Transform.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using static CyberDot.Encoding.Base91.Constants; + +namespace CyberDot.Encoding.Base91 +{ + /// + /// Streams raw bytes into their Base91 representation, serialised using whichever + /// is supplied to the constructor (UTF-8 by default). Since the + /// Base91 alphabet is pure ASCII, the resulting text is equally valid under any of them. + /// Intended for use with . + /// + public class ToBase91Transform : ICryptoTransform + { + private readonly System.Text.Encoding encoding; + + // Bit accumulator state, carried across TransformBlock calls: a pair of characters + // holds 13 or 14 input bits, so almost every input byte leaves a partial pair pending. + private uint queue; + private int numBits; + + public ToBase91Transform(System.Text.Encoding encoding = null) + { + this.encoding = encoding ?? new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + OutputBlockSize = this.encoding.GetMaxByteCount(2); + } + + public bool CanReuseTransform => true; + public bool CanTransformMultipleBlocks => true; + public int InputBlockSize => 1; + public int OutputBlockSize { get; } + + public void Dispose() { } + + public int TransformBlock(byte[] inputBuffer, int inputOffset, int inputCount, byte[] outputBuffer, int outputOffset) + { + var output = ProcessBytes(inputBuffer, inputOffset, inputCount, isFinal: false); + Array.Copy(output, 0, outputBuffer, outputOffset, output.Length); + return output.Length; + } + + public byte[] TransformFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount) + { + var result = ProcessBytes(inputBuffer, inputOffset, inputCount, isFinal: true); + queue = 0; + numBits = 0; + return result; + } + + private byte[] ProcessBytes(byte[] inputBuffer, int inputOffset, int inputCount, bool isFinal) + { + if (inputBuffer == null) + { + throw new ArgumentNullException(nameof(inputBuffer)); + } + + if (inputOffset < 0 || inputCount < 0 || inputOffset + inputCount > inputBuffer.Length) + { + throw new ArgumentOutOfRangeException(nameof(inputCount)); + } + + var chars = new List(inputCount * 2); + var end = inputOffset + inputCount; + + for (var i = inputOffset; i < end; i++) + { + queue |= (uint)inputBuffer[i] << numBits; + numBits += BitsPerByte; + + if (numBits > QueueFlushThreshold) + { + var bits = Base91.NextPairBits(queue, out var v); + chars.Add(Base91.Alphabet[v % Radix]); + chars.Add(Base91.Alphabet[v / Radix]); + queue >>= bits; + numBits -= bits; + } + } + + if (isFinal && numBits > 0) + { + chars.Add(Base91.Alphabet[queue % Radix]); + if (!Base91.FitsOneChar(numBits, queue)) + { + chars.Add(Base91.Alphabet[queue / Radix]); + } + } + + return chars.Count == 0 ? Array.Empty() : encoding.GetBytes(chars.ToArray()); + } + } +} From 7e854a30881b2f8fde4cf53e364461a2084238d8 Mon Sep 17 00:00:00 2001 From: cyberdot Date: Sat, 19 Sep 2026 19:18:22 +0100 Subject: [PATCH 2/2] Build / deploy functionality for base91 encoding --- .github/workflows/ci.yml | 3 ++- .github/workflows/release-base91.yml | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release-base91.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dc53f2..8675f04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,7 @@ jobs: for project in \ src/CyberDot.Encoding.Base65536/CyberDot.Encoding.Base65536.csproj \ src/CyberDot.Encoding.Base2048/CyberDot.Encoding.Base2048.csproj \ - src/CyberDot.Encoding.Base84/CyberDot.Encoding.Base84.csproj; do + src/CyberDot.Encoding.Base84/CyberDot.Encoding.Base84.csproj \ + src/CyberDot.Encoding.Base91/CyberDot.Encoding.Base91.csproj; do dotnet pack "$project" --configuration Debug --no-build --output "${{ runner.temp }}/nupkg-verify" done diff --git a/.github/workflows/release-base91.yml b/.github/workflows/release-base91.yml new file mode 100644 index 0000000..a410e4c --- /dev/null +++ b/.github/workflows/release-base91.yml @@ -0,0 +1,22 @@ +name: Release CyberDot.Encoding.Base91 + +# Manually-triggered release for the Base91 package. See _release-package.yml for +# what this actually does. + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version override (e.g. 1.2.0). Leave empty to use the version in the csproj.' + required: false + type: string + +jobs: + release: + permissions: + contents: write + uses: ./.github/workflows/_release-package.yml + with: + project-path: src/CyberDot.Encoding.Base91/CyberDot.Encoding.Base91.csproj + tag-prefix: base91 + version: ${{ inputs.version }}