From f2853fb8414df76448aa07fddd48cd8aca647d66 Mon Sep 17 00:00:00 2001 From: cyberdot Date: Tue, 15 Sep 2026 17:41:02 +0100 Subject: [PATCH 1/2] Base84 implementation in C# --- .gitattributes | 11 + CyberDot.Encoding.sln | 28 +- .../Base84Tests.cs | 218 +++++++++++++ .../CyberDot.Encoding.Base84.Tests.csproj | 26 ++ .../StreamTransformTests.cs | 288 ++++++++++++++++++ src/CyberDot.Encoding.Base84/Base84.cs | 201 ++++++++++++ src/CyberDot.Encoding.Base84/Constants.cs | 49 +++ .../CyberDot.Encoding.Base84.csproj | 24 ++ .../FromBase84Transform.cs | 134 ++++++++ src/CyberDot.Encoding.Base84/README.md | 44 +++ .../ToBase84Transform.cs | 106 +++++++ 11 files changed, 1128 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100644 src/CyberDot.Encoding.Base84.Tests/Base84Tests.cs create mode 100644 src/CyberDot.Encoding.Base84.Tests/CyberDot.Encoding.Base84.Tests.csproj create mode 100644 src/CyberDot.Encoding.Base84.Tests/StreamTransformTests.cs create mode 100644 src/CyberDot.Encoding.Base84/Base84.cs create mode 100644 src/CyberDot.Encoding.Base84/Constants.cs create mode 100644 src/CyberDot.Encoding.Base84/CyberDot.Encoding.Base84.csproj create mode 100644 src/CyberDot.Encoding.Base84/FromBase84Transform.cs create mode 100644 src/CyberDot.Encoding.Base84/README.md create mode 100644 src/CyberDot.Encoding.Base84/ToBase84Transform.cs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..25f98ef --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# These directories hold test fixtures captured verbatim from reference +# implementations - raw encoded/decoded bytes, not human-authored text. Some of those +# byte sequences happen to contain a lone LF (0x0A); without this, Git's line-ending +# conversion (autocrlf) can silently rewrite it to CRLF on checkout, corrupting the +# fixture without the tracked blob ever changing. Disable text conversion for them. +src/CyberDot.Encoding.Base2048.Tests/test-data/** -text +src/CyberDot.Encoding.Base65536.Tests/Bad/** -text +src/CyberDot.Encoding.Base65536.Tests/DoubledBytes/** -text +src/CyberDot.Encoding.Base65536.Tests/IgnoreGarbage/** -text +src/CyberDot.Encoding.Base65536.Tests/Pairs/** -text +src/CyberDot.Encoding.Base65536.Tests/SingleBytes/** -text diff --git a/CyberDot.Encoding.sln b/CyberDot.Encoding.sln index 3d5b41d..5aa16dd 100644 --- a/CyberDot.Encoding.sln +++ b/CyberDot.Encoding.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 18 -VisualStudioVersion = 18.9.12105.275 stable +VisualStudioVersion = 18.9.12105.275 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CyberDot.Encoding.Base65536", "src\CyberDot.Encoding.Base65536\CyberDot.Encoding.Base65536.csproj", "{C021E149-EB05-4EF0-9A4C-78A0808AC413}" EndProject @@ -11,6 +11,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CyberDot.Encoding.Base2048" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CyberDot.Encoding.Base2048.Tests", "src\CyberDot.Encoding.Base2048.Tests\CyberDot.Encoding.Base2048.Tests.csproj", "{287EE26D-CAE7-4AA3-93C5-4A6B2C8A9D05}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CyberDot.Encoding.Base84", "src\CyberDot.Encoding.Base84\CyberDot.Encoding.Base84.csproj", "{9551EC7C-8F4A-462A-AABC-50D1FD1E60B5}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Base2048", "Base2048", "{7F2B409E-7822-462C-8152-0DE5BEE42CAD}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Base65536", "Base65536", "{EF960794-148F-4CC2-A1B6-4001FA1A9095}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CyberDot.Encoding.Base84.Tests", "src\CyberDot.Encoding.Base84.Tests\CyberDot.Encoding.Base84.Tests.csproj", "{3762FA86-26D7-4037-B7CF-E5CA94F03D8B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Base84", "Base84", "{4C6BEADC-6B8C-425C-9EC4-559597A7915C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -33,10 +43,26 @@ Global {287EE26D-CAE7-4AA3-93C5-4A6B2C8A9D05}.Debug|Any CPU.Build.0 = Debug|Any CPU {287EE26D-CAE7-4AA3-93C5-4A6B2C8A9D05}.Release|Any CPU.ActiveCfg = Release|Any CPU {287EE26D-CAE7-4AA3-93C5-4A6B2C8A9D05}.Release|Any CPU.Build.0 = Release|Any CPU + {9551EC7C-8F4A-462A-AABC-50D1FD1E60B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9551EC7C-8F4A-462A-AABC-50D1FD1E60B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9551EC7C-8F4A-462A-AABC-50D1FD1E60B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9551EC7C-8F4A-462A-AABC-50D1FD1E60B5}.Release|Any CPU.Build.0 = Release|Any CPU + {3762FA86-26D7-4037-B7CF-E5CA94F03D8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3762FA86-26D7-4037-B7CF-E5CA94F03D8B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3762FA86-26D7-4037-B7CF-E5CA94F03D8B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3762FA86-26D7-4037-B7CF-E5CA94F03D8B}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {C021E149-EB05-4EF0-9A4C-78A0808AC413} = {EF960794-148F-4CC2-A1B6-4001FA1A9095} + {F4232660-0EA0-432E-8713-F4612E88AA80} = {EF960794-148F-4CC2-A1B6-4001FA1A9095} + {C7494689-9E71-4F15-883E-027E244E3E6D} = {7F2B409E-7822-462C-8152-0DE5BEE42CAD} + {287EE26D-CAE7-4AA3-93C5-4A6B2C8A9D05} = {7F2B409E-7822-462C-8152-0DE5BEE42CAD} + {9551EC7C-8F4A-462A-AABC-50D1FD1E60B5} = {4C6BEADC-6B8C-425C-9EC4-559597A7915C} + {3762FA86-26D7-4037-B7CF-E5CA94F03D8B} = {4C6BEADC-6B8C-425C-9EC4-559597A7915C} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9D040A49-47AE-4503-B1C2-7D3E82AF66AB} EndGlobalSection diff --git a/src/CyberDot.Encoding.Base84.Tests/Base84Tests.cs b/src/CyberDot.Encoding.Base84.Tests/Base84Tests.cs new file mode 100644 index 0000000..3181907 --- /dev/null +++ b/src/CyberDot.Encoding.Base84.Tests/Base84Tests.cs @@ -0,0 +1,218 @@ +using AwesomeAssertions; + +namespace CyberDot.Encoding.Base84.Tests +{ + public class Base84Tests + { + // Vectors below are taken from the reference Zig implementation's test suite. + [Fact] + public void Should_encode_and_decode_known_vectors() + { + AssertVector("", ""); + AssertVector("M", "]A"); + AssertVector("Ma", "tsD"); + AssertVector("Man", "pRRM"); + AssertVector("Man ", ";dA^K"); + AssertVector("Hello, World!", "s@Etk'#Qedrxz+hhA"); + AssertVector("The quick brown fox jumps over the lazy dog.", + "wBB]KdJ}phb-#tmbNB^K!!J_K^3h=l_pj[nJXJLnkB3kkhkT_KM}r1P"); + } + + [Fact] + public void Should_encode_and_decode_runs_of_zero_bytes() + { + AssertVector(new byte[] { 0x00 }, "AA"); + AssertVector(new byte[] { 0x00, 0x00 }, "AAA"); + AssertVector(new byte[] { 0x00, 0x00, 0x00 }, "AAAA"); + AssertVector(new byte[] { 0x00, 0x00, 0x00, 0x00 }, "AAAAA"); + AssertVector(new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00 }, "AAAAAAA"); + } + + [Fact] + public void Should_encode_and_decode_runs_of_ff_bytes() + { + AssertVector(new byte[] { 0xff }, "DD"); + AssertVector(new byte[] { 0xff, 0xff }, "PYJ"); + AssertVector(new byte[] { 0xff, 0xff, 0xff }, "#8Zc"); + AssertVector(new byte[] { 0xff, 0xff, 0xff, 0xff }, "rxQLrB"); + AssertVector(new byte[] { 0xff, 0xff, 0xff, 0xff, 0xff }, "rxQLrHG"); + } + + [Fact] + public void Should_encode_and_decode_mixed_bytes() + { + AssertVector(new byte[] { 0x00, 0xe0, 0xff, 0x01 }, "AYy4A"); + AssertVector( + new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + "8%LBB${'eC(No8D-dMGF"); + } + + [Theory] + [InlineData("AA/")] + [InlineData("A A")] + [InlineData("AA\x00")] + [InlineData("AA\xff")] + [InlineData("AA<")] + public void Should_throw_on_unrecognised_character(string input) + { + var act = () => Base84.Decode(input); + + act.Should().Throw(); + } + + [Theory] + [InlineData("A")] + [InlineData("AE")] + [InlineData("AAK")] + [InlineData("AAAd")] + [InlineData("rxQLr")] + [InlineData("oi'-o")] + [InlineData("AAAAAA")] + [InlineData("rxQLrrxQLrrxQLrrxQLrrxQLrrxQLrrxQLrAA")] + public void Should_throw_on_invalid_padding(string input) + { + var act = () => Base84.Decode(input); + + act.Should().Throw(); + } + + [Fact] + public void Should_throw_on_null_input() + { + var encodeAct = () => Base84.Encode(null); + var decodeAct = () => Base84.Decode(null); + + encodeAct.Should().Throw(); + decodeAct.Should().Throw(); + } + + [Fact] + public void Should_round_trip_all_single_bytes() + { + for (var b = 0; b <= 255; b++) + { + var data = new[] { (byte)b }; + + var encoded = Base84.Encode(data); + Base84.Decode(encoded).Should().BeEquivalentTo(data); + } + } + + [Fact] + public void Should_round_trip_all_byte_pairs() + { + for (var a = 0; a <= 255; a++) + { + for (var b = 0; b <= 255; b++) + { + var data = new[] { (byte)a, (byte)b }; + + var encoded = Base84.Encode(data); + Base84.Decode(encoded).Should().BeEquivalentTo(data); + } + } + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(17)] + [InlineData(31)] + [InlineData(32)] + [InlineData(33)] + [InlineData(256)] + public void Should_round_trip_random_data(int length) + { + var data = new byte[length]; + new Random(length).NextBytes(data); + + var encoded = Base84.Encode(data); + Base84.Decode(encoded).Should().BeEquivalentTo(data); + } + + [Fact] + public void Should_round_trip_many_random_lengths() + { + var random = new Random(12345); + var buffer = new byte[256]; + + for (var i = 0; i < 5000; i++) + { + var len = random.Next(0, buffer.Length + 1); + random.NextBytes(buffer); + var data = new byte[len]; + Array.Copy(buffer, data, len); + + var encoded = Base84.Encode(data); + Base84.Decode(encoded).Should().BeEquivalentTo(data); + } + } + + private const string AlphabetChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&'()+,-;=@[]^_`{}~"; + + [Fact] + public void Should_reject_altered_encoded_data_or_accept_it_and_round_trip() + { + // Any mutation of a validly-encoded string must either be rejected as invalid + // padding, or - if it happens to still be well-formed - round trip cleanly + // through a fresh encode. + var random = new Random(999); + var buffer = new byte[32]; + + for (var i = 0; i < 5000; i++) + { + var len = random.Next(0, buffer.Length + 1); + random.NextBytes(buffer); + var data = new byte[len]; + Array.Copy(buffer, data, len); + + var encodedChars = Base84.Encode(data).ToCharArray(); + var alphabetChar = AlphabetChars[random.Next(AlphabetChars.Length)]; + + switch (random.Next(3)) + { + case 0 when encodedChars.Length > 0: + encodedChars[random.Next(encodedChars.Length)] = alphabetChar; + break; + case 1: + Array.Resize(ref encodedChars, encodedChars.Length + 1); + encodedChars[encodedChars.Length - 1] = alphabetChar; + break; + case 2 when encodedChars.Length > 0: + Array.Resize(ref encodedChars, encodedChars.Length - 1); + break; + } + + var altered = new string(encodedChars); + + byte[] decoded; + try + { + decoded = Base84.Decode(altered); + } + catch (ArgumentException) + { + continue; + } + + Base84.Encode(decoded).Should().Be(altered); + } + } + + private static void AssertVector(string decoded, string encoded) + { + AssertVector(System.Text.Encoding.UTF8.GetBytes(decoded), encoded); + } + + private static void AssertVector(byte[] decoded, string encoded) + { + Base84.Encode(decoded).Should().Be(encoded); + Base84.Decode(encoded).Should().BeEquivalentTo(decoded); + } + } +} diff --git a/src/CyberDot.Encoding.Base84.Tests/CyberDot.Encoding.Base84.Tests.csproj b/src/CyberDot.Encoding.Base84.Tests/CyberDot.Encoding.Base84.Tests.csproj new file mode 100644 index 0000000..2b5fa8a --- /dev/null +++ b/src/CyberDot.Encoding.Base84.Tests/CyberDot.Encoding.Base84.Tests.csproj @@ -0,0 +1,26 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/CyberDot.Encoding.Base84.Tests/StreamTransformTests.cs b/src/CyberDot.Encoding.Base84.Tests/StreamTransformTests.cs new file mode 100644 index 0000000..398f8d8 --- /dev/null +++ b/src/CyberDot.Encoding.Base84.Tests/StreamTransformTests.cs @@ -0,0 +1,288 @@ +using System.Security.Cryptography; +using AwesomeAssertions; +using TextEncoding = System.Text.Encoding; + +namespace CyberDot.Encoding.Base84.Tests +{ + public class StreamTransformTests + { + private static readonly int[] ChunkSizes = [1, 2, 3, 4, 5, 7, 13, 64]; + + // Base84's alphabet is pure ASCII, so the encoded text is equally valid under any + // of these three encodings - the transforms support all of them. + private static readonly TextEncoding[] SafeEncodings = [TextEncoding.UTF8, TextEncoding.Unicode, TextEncoding.UTF32]; + + private static readonly (byte[] Decoded, string Encoded)[] Vectors = + { + (System.Array.Empty(), ""), + (TextEncoding.UTF8.GetBytes("M"), "]A"), + (TextEncoding.UTF8.GetBytes("Ma"), "tsD"), + (TextEncoding.UTF8.GetBytes("Man"), "pRRM"), + (TextEncoding.UTF8.GetBytes("Man "), ";dA^K"), + (TextEncoding.UTF8.GetBytes("Hello, World!"), "s@Etk'#Qedrxz+hhA"), + (new byte[] { 0xff, 0xff, 0xff, 0xff, 0xff }, "rxQLrHG"), + ( + TextEncoding.UTF8.GetBytes("The quick brown fox jumps over the lazy dog."), + "wBB]KdJ}phb-#tmbNB^K!!J_K^3h=l_pj[nJXJLnkB3kkhkT_KM}r1P" + ), + }; + + private static byte[] EncodeViaCryptoStream(byte[] data, TextEncoding? encoding = null) + { + using var ms = new MemoryStream(); + using (var cs = new CryptoStream(ms, new ToBase84Transform(encoding), CryptoStreamMode.Write, leaveOpen: true)) + { + cs.Write(data, 0, data.Length); + cs.FlushFinalBlock(); + } + + return ms.ToArray(); + } + + private static byte[] DecodeViaCryptoStream(byte[] bytes, TextEncoding? encoding = null) + { + using var ms = new MemoryStream(bytes); + using var cs = new CryptoStream(ms, new FromBase84Transform(encoding), CryptoStreamMode.Read); + using var outMs = new MemoryStream(); + cs.CopyTo(outMs); + return outMs.ToArray(); + } + + private static byte[] DriveTransform(ICryptoTransform transform, byte[] input, int chunkSize) + { + using var outMs = new MemoryStream(); + var i = 0; + + while (input.Length - i > chunkSize) + { + var buffer = new byte[chunkSize * 8 + 32]; + var written = transform.TransformBlock(input, i, chunkSize, buffer, 0); + outMs.Write(buffer, 0, written); + i += chunkSize; + } + + var final = transform.TransformFinalBlock(input, i, input.Length - i); + outMs.Write(final, 0, final.Length); + + return outMs.ToArray(); + } + + private static byte[] AsBytes(string value, TextEncoding? encoding = null) => (encoding ?? TextEncoding.UTF8).GetBytes(value); + + // --- Round trip against the known vectors (default: UTF-8) ----------------------------------------------- + + [Fact] + public void Should_stream_encode_known_vectors() + { + foreach (var (decoded, encoded) in Vectors) + { + EncodeViaCryptoStream(decoded).Should().BeEquivalentTo(AsBytes(encoded)); + } + } + + [Fact] + public void Should_stream_decode_known_vectors() + { + foreach (var (decoded, encoded) in Vectors) + { + DecodeViaCryptoStream(AsBytes(encoded)).Should().BeEquivalentTo(decoded); + } + } + + [Theory] + [InlineData("AA/")] + [InlineData("A A")] + public void Should_throw_on_unrecognised_character_while_streaming(string input) + { + var act = () => DecodeViaCryptoStream(AsBytes(input)); + + act.Should().Throw(); + } + + [Theory] + [InlineData("A")] + [InlineData("AE")] + [InlineData("AAK")] + [InlineData("AAAAAA")] + public void Should_throw_on_invalid_padding_while_streaming(string input) + { + var act = () => DecodeViaCryptoStream(AsBytes(input)); + + act.Should().Throw(); + } + + // --- Round trip across all three "safe" encodings --------------------------------------------------------- + + [Fact] + public void Should_round_trip_known_vectors_across_all_safe_encodings() + { + foreach (var encoding in SafeEncodings) + { + foreach (var (decoded, encoded) in Vectors) + { + EncodeViaCryptoStream(decoded, encoding) + .Should().BeEquivalentTo(AsBytes(encoded, encoding), $"encoding {encoding.EncodingName}"); + + DecodeViaCryptoStream(AsBytes(encoded, encoding), encoding) + .Should().BeEquivalentTo(decoded, $"encoding {encoding.EncodingName}"); + } + } + } + + // --- Chunked feeding: exercises bit-accumulator / group-buffering across calls ---------------------------- + + [Fact] + public void Should_encode_correctly_regardless_of_chunk_size() + { + foreach (var (decoded, encoded) in Vectors) + { + foreach (var encoding in SafeEncodings) + { + var expectedBytes = AsBytes(encoded, encoding); + + foreach (var chunkSize in ChunkSizes) + { + DriveTransform(new ToBase84Transform(encoding), decoded, chunkSize) + .Should().BeEquivalentTo(expectedBytes, $"chunk size {chunkSize}, encoding {encoding.EncodingName}"); + } + } + } + } + + [Fact] + public void Should_decode_correctly_regardless_of_chunk_size() + { + foreach (var (decoded, encoded) in Vectors) + { + foreach (var encoding in SafeEncodings) + { + var inputBytes = AsBytes(encoded, encoding); + + foreach (var chunkSize in ChunkSizes) + { + DriveTransform(new FromBase84Transform(encoding), inputBytes, chunkSize) + .Should().BeEquivalentTo(decoded, $"chunk size {chunkSize}, encoding {encoding.EncodingName}"); + } + } + } + } + + // --- Round trip fuzzing over random data of varying lengths ------------------------------------------------ + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(17)] + [InlineData(256)] + [InlineData(1000)] + [InlineData(4097)] + public void Should_round_trip_random_data_through_crypto_stream(int length) + { + var data = new byte[length]; + new Random(length).NextBytes(data); + + foreach (var encoding in SafeEncodings) + { + var encoded = EncodeViaCryptoStream(data, encoding); + encoded.Should().BeEquivalentTo(AsBytes(Base84.Encode(data), encoding), $"encoding {encoding.EncodingName}"); + + var decoded = DecodeViaCryptoStream(encoded, encoding); + decoded.Should().BeEquivalentTo(data, $"encoding {encoding.EncodingName}"); + } + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(17)] + [InlineData(256)] + public void Should_round_trip_random_data_with_odd_chunk_sizes(int length) + { + var data = new byte[length]; + new Random(length + 1).NextBytes(data); + + foreach (var encoding in SafeEncodings) + { + foreach (var chunkSize in ChunkSizes) + { + var encoded = DriveTransform(new ToBase84Transform(encoding), data, chunkSize); + var decoded = DriveTransform(new FromBase84Transform(encoding), encoded, chunkSize); + + decoded.Should().BeEquivalentTo(data, $"chunk size {chunkSize}, encoding {encoding.EncodingName}, length {length}"); + } + } + } + + // --- Transform metadata -------------------------------------------------------------------------------- + + [Theory] + [MemberData(nameof(SafeEncodingsMemberData))] + public void Should_expose_expected_metadata_on_encode(TextEncoding encoding) + { + var transform = new ToBase84Transform(encoding); + + transform.InputBlockSize.Should().Be(1); + transform.OutputBlockSize.Should().BeGreaterThanOrEqualTo(5); + transform.CanTransformMultipleBlocks.Should().BeTrue(); + transform.CanReuseTransform.Should().BeTrue(); + } + + [Theory] + [MemberData(nameof(SafeEncodingsMemberData))] + public void Should_expose_expected_metadata_on_decode(TextEncoding encoding) + { + var transform = new FromBase84Transform(encoding); + + transform.InputBlockSize.Should().Be(1); + transform.OutputBlockSize.Should().Be(1); + transform.CanTransformMultipleBlocks.Should().BeTrue(); + transform.CanReuseTransform.Should().BeTrue(); + } + + [Fact] + public void Should_default_to_utf8_when_no_encoding_supplied() + { + var data = new byte[] { 1, 2, 3, 4, 5 }; + + EncodeViaCryptoStream(data).Should().BeEquivalentTo(AsBytes(Base84.Encode(data), TextEncoding.UTF8)); + } + + public static TheoryData SafeEncodingsMemberData() + { + var data = new TheoryData(); + foreach (var encoding in SafeEncodings) data.Add(encoding); + return data; + } + + [Fact] + public void Should_reuse_transform_instances_across_multiple_streams() + { + var toTransform = new ToBase84Transform(); + var fromTransform = new FromBase84Transform(); + + var first = new byte[] { 1, 2, 3 }; + var second = new byte[] { 4, 5, 6, 7 }; + + foreach (var data in new[] { first, second }) + { + using var ms = new MemoryStream(); + using (var cs = new CryptoStream(ms, toTransform, CryptoStreamMode.Write, leaveOpen: true)) + { + cs.Write(data, 0, data.Length); + cs.FlushFinalBlock(); + } + + using var decodedMs = new MemoryStream(ms.ToArray()); + using var decodeCs = new CryptoStream(decodedMs, fromTransform, CryptoStreamMode.Read); + using var outMs = new MemoryStream(); + decodeCs.CopyTo(outMs); + + outMs.ToArray().Should().BeEquivalentTo(data); + } + } + } +} diff --git a/src/CyberDot.Encoding.Base84/Base84.cs b/src/CyberDot.Encoding.Base84/Base84.cs new file mode 100644 index 0000000..dba23d2 --- /dev/null +++ b/src/CyberDot.Encoding.Base84/Base84.cs @@ -0,0 +1,201 @@ +using System; +using System.Collections.Generic; +using System.Text; +using static CyberDot.Encoding.Base84.Constants; + +namespace CyberDot.Encoding.Base84 +{ + /// + /// Base84 is a binary-to-text encoding that uses 84 ASCII characters, all safe for file + /// names on macOS, Linux and Windows. It works like Base91: every five characters can + /// hold 31 or 32 input bits, so encoded output is about 25% larger than the input, + /// against 33% for Base64. + /// + /// The alphabet has every printable ASCII character except space, the period, and the + /// nine characters that Windows rejects in file names: < > : " / \ | ? * + /// + public static class Base84 + { + private const string AlphabetChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&'()+,-;=@[]^_`{}~"; + private const byte None = 0xff; + + internal static readonly char[] Alphabet = AlphabetChars.ToCharArray(); + internal static readonly byte[] InverseMap = BuildInverseMap(); + + private static byte[] BuildInverseMap() + { + var map = new byte[256]; + for (var i = 0; i < map.Length; i++) + { + map[i] = None; + } + + for (var i = 0; i < Alphabet.Length; i++) + { + map[Alphabet[i]] = (byte)i; + } + + return map; + } + + public static string Encode(byte[] data) + { + if (data == null) + { + throw new ArgumentNullException(nameof(data)); + } + + var result = new StringBuilder(data.Length * GroupChars / 4 + GroupChars); + ulong acc = 0; + var numBits = 0; + + foreach (var x in data) + { + acc |= (ulong)x << numBits; + numBits += BitsPerByte; + + if (numBits > GroupBits) + { + var low = (uint)acc; + var bits = GroupBitsOf(low); + var v = bits > GroupBits ? low : low & GroupMask; + acc >>= bits; + numBits -= bits; + WriteDigits(result, v, GroupChars); + } + } + + if (numBits > 0) + { + var tail = (uint)acc; + var chars = FitsOneChar(numBits, tail) ? 1 : TailChars[numBits]; + WriteDigits(result, tail, chars); + } + + return result.ToString(); + } + + /// + /// Decodes a Base84 encoded string back into a byte slice. + /// Decoding is strict: it rejects any ending the encoder cannot produce. + /// + public static byte[] Decode(string data) + { + if (data == null) + { + throw new ArgumentNullException(nameof(data)); + } + + var output = new List(data.Length * GroupBits / (GroupChars * BitsPerByte) + GroupChars); + ulong acc = 0; + var numBits = 0; + var bits = 0; + + var i = 0; + while (i < data.Length) + { + var chunkLen = Math.Min(GroupChars, data.Length - i); + var v = ReadChunk(data, i, chunkLen); + bits = chunkLen == GroupChars ? GroupBitsOf(v) : TailBitsOf(chunkLen, numBits, v); + + acc |= (ulong)v << numBits; + numBits += bits; + + while (numBits >= BitsPerByte) + { + output.Add((byte)acc); + acc >>= BitsPerByte; + numBits -= BitsPerByte; + } + + i += GroupChars; + } + + if (data.Length > 0 && data.Length % GroupChars == 0) + { + // Reject a final group with padding bits set, or one that fits in fewer characters. + if (acc != 0 || bits - numBits <= MaxTailBits) + { + throw new ArgumentException("Invalid Base84 padding."); + } + } + + return output.ToArray(); + } + + // The number of input bits that a group of five characters holds for this value. + internal static int GroupBitsOf(uint v) + { + return (v & GroupMask) < ExtraBitLimit ? GroupBits + 1 : GroupBits; + } + + // Whether the encoder writes a single character for this tail. + internal static bool FitsOneChar(int bits, uint tail) + { + return bits <= 7 && tail < Radix; + } + + // The number of input bits in a short final chunk, or an error for a chunk the encoder never writes. + internal static int TailBitsOf(int chars, int pending, uint v) + { + var bits = TailBits[chars][pending]; + if (bits == 0 || v >> bits != 0) + { + throw new ArgumentException("Invalid Base84 padding."); + } + + if (chars > 1 && FitsOneChar(bits, v)) + { + throw new ArgumentException("Invalid Base84 padding."); + } + + return bits; + } + + internal static uint ReadChunk(string data, int start, int length) + { + uint v = 0; + uint place = 1; + for (var i = 0; i < length; i++) + { + v += Digit(data[start + i]) * place; + place *= Radix; + } + + return v; + } + + internal static uint ReadChunk(char[] chunk, int length) + { + uint v = 0; + uint place = 1; + for (var i = 0; i < length; i++) + { + v += Digit(chunk[i]) * place; + place *= Radix; + } + + return v; + } + + internal static uint Digit(char c) + { + var value = c < InverseMap.Length ? InverseMap[c] : None; + if (value == None) + { + throw new ArgumentException("Unrecognised Base84 character: '" + c + "'."); + } + + return value; + } + + private static void WriteDigits(StringBuilder sb, uint value, int count) + { + for (var i = 0; i < count; i++) + { + sb.Append(Alphabet[value % Radix]); + value /= Radix; + } + } + } +} diff --git a/src/CyberDot.Encoding.Base84/Constants.cs b/src/CyberDot.Encoding.Base84/Constants.cs new file mode 100644 index 0000000..4562e82 --- /dev/null +++ b/src/CyberDot.Encoding.Base84/Constants.cs @@ -0,0 +1,49 @@ +namespace CyberDot.Encoding.Base84 +{ + // Numeric tables ported from the reference Zig implementation (RefImplementation.txt). + // Base84 packs input bits into groups of five characters instead of a fixed number of + // bits per character: five base-84 digits (84^5 ≈ 2^31.96) can always hold 31 input + // bits, and can often stretch to hold 32. These tables answer the two questions that + // arise from that variable width: how many characters does a short final chunk need, + // and how many bits does a short final chunk of "n" characters actually hold. + internal static class Constants + { + public const int BitsPerByte = 8; + public const int Radix = 84; + + // A group of five characters is the natural encoding unit. + public const int GroupChars = 5; + + // The number of input bits that five characters can always hold: floor(log2(84^5)). + public const int GroupBits = 31; + public const uint GroupMask = (1u << GroupBits) - 1; + + // A group holds a 32nd bit whenever its low 31 bits, taken alone, are below this + // limit: 84^5 - 2^31. Below the limit, the full 32-bit value still fits under 84^5. + public const uint ExtraBitLimit = 2034635776; + + // The most bits that four characters can ever hold (floor(log2(84^4))). Used to spot + // a final five-character group that could have been written with fewer characters. + public const int MaxTailBits = 25; + + // tail_chars[bits]: the number of characters the encoder writes for a tail of + // "bits" pending input bits (0-31). This is the smallest character count whose + // guaranteed capacity covers "bits". + public static readonly int[] TailChars = + { + 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, + }; + + // tail_bits[chars][pending]: the number of input bits held by a short final chunk of + // "chars" characters, given "pending" bits (0-7) were already waiting in the byte + // accumulator. Zero marks a (chars, pending) combination the encoder never produces. + public static readonly int[][] TailBits = + { + new[] { 0, 0, 0, 0, 0, 0, 0, 0 }, + new[] { 0, 7, 6, 5, 4, 3, 2, 1 }, + new[] { 8, 7, 0, 0, 12, 11, 10, 9 }, + new[] { 16, 15, 14, 13, 0, 19, 18, 17 }, + new[] { 24, 23, 22, 21, 20, 0, 0, 25 }, + }; + } +} diff --git a/src/CyberDot.Encoding.Base84/CyberDot.Encoding.Base84.csproj b/src/CyberDot.Encoding.Base84/CyberDot.Encoding.Base84.csproj new file mode 100644 index 0000000..cd0f58c --- /dev/null +++ b/src/CyberDot.Encoding.Base84/CyberDot.Encoding.Base84.csproj @@ -0,0 +1,24 @@ + + + + netstandard1.6 + True + 1.1.0 + CyberDot + CyberDot + An implementation of Base84 binary-to-text encoding in C#, including streaming encode/decode functionality via ICryptoTransform/CryptoStream. + Base84, Encoding + MIT + README.md + https://github.com/cyberdot/CyberDot.Encoding + https://github.com/cyberdot/CyberDot.Encoding + git + true + snupkg + + + + + + + diff --git a/src/CyberDot.Encoding.Base84/FromBase84Transform.cs b/src/CyberDot.Encoding.Base84/FromBase84Transform.cs new file mode 100644 index 0000000..413e49c --- /dev/null +++ b/src/CyberDot.Encoding.Base84/FromBase84Transform.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using static CyberDot.Encoding.Base84.Constants; + +namespace CyberDot.Encoding.Base84 +{ + /// + /// Streams a Base84 sequence back into the original raw bytes. Since the Base84 + /// alphabet is pure ASCII, the input text may validly have been serialised with any + /// ; pass the matching one to the constructor (UTF-8 by default). + /// Intended for use with . + /// + public class FromBase84Transform : ICryptoTransform + { + private readonly Decoder decoder; + + // Characters are only ever decoded five at a time, so at most four can be pending + // between calls, waiting on the rest of their group. + private readonly char[] pending = new char[GroupChars]; + private int pendingCount; + + // Byte accumulator state, carried across TransformBlock calls. + private ulong acc; + private int numBits; + + // The bit width of the most recently decoded chunk, needed - together with + // totalChars - to validate the padding of a final full group once the stream ends. + private int bits; + private long totalChars; + + public FromBase84Transform(System.Text.Encoding encoding = null) + { + decoder = (encoding ?? new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true)).GetDecoder(); + } + + public bool CanReuseTransform => true; + public bool CanTransformMultipleBlocks => true; + public int InputBlockSize => 1; + public int OutputBlockSize => 1; + + public void Dispose() { } + + public int TransformBlock(byte[] inputBuffer, int inputOffset, int inputCount, byte[] outputBuffer, int outputOffset) + { + var output = ProcessBytes(inputBuffer, inputOffset, inputCount, isFinal: false); + Array.Copy(output, 0, outputBuffer, outputOffset, output.Length); + return output.Length; + } + + public byte[] TransformFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount) + { + var result = ProcessBytes(inputBuffer, inputOffset, inputCount, isFinal: true); + Reset(); + return result; + } + + private void Reset() + { + decoder.Reset(); + pendingCount = 0; + acc = 0; + numBits = 0; + bits = 0; + totalChars = 0; + } + + // The Decoder buffers any incomplete multibyte sequence internally across calls + // (and, with a throwing Encoding, rejects malformed/truncated input), so we only + // need to track our own group-buffering and bit-accumulator state here. + private byte[] ProcessBytes(byte[] inputBuffer, int inputOffset, int inputCount, bool isFinal) + { + if (inputBuffer == null) + { + throw new ArgumentNullException(nameof(inputBuffer)); + } + + var maxChars = decoder.GetCharCount(inputBuffer, inputOffset, inputCount, isFinal); + var chars = maxChars == 0 ? Array.Empty() : new char[maxChars]; + var charCount = decoder.GetChars(inputBuffer, inputOffset, inputCount, chars, 0, isFinal); + + var output = new List(charCount); + + for (var i = 0; i < charCount; i++) + { + pending[pendingCount++] = chars[i]; + totalChars++; + + if (pendingCount == GroupChars) + { + HandleChunk(GroupChars, isFullGroup: true, output); + pendingCount = 0; + } + } + + if (isFinal) + { + if (pendingCount > 0) + { + HandleChunk(pendingCount, isFullGroup: false, output); + pendingCount = 0; + } + + if (totalChars > 0 && totalChars % GroupChars == 0) + { + // Reject a final group with padding bits set, or one that fits in fewer characters. + if (acc != 0 || bits - numBits <= MaxTailBits) + { + throw new ArgumentException("Invalid Base84 padding."); + } + } + } + + return output.ToArray(); + } + + private void HandleChunk(int chunkLen, bool isFullGroup, List output) + { + var v = Base84.ReadChunk(pending, chunkLen); + bits = isFullGroup ? Base84.GroupBitsOf(v) : Base84.TailBitsOf(chunkLen, numBits, v); + + acc |= (ulong)v << numBits; + numBits += bits; + + while (numBits >= BitsPerByte) + { + output.Add((byte)acc); + acc >>= BitsPerByte; + numBits -= BitsPerByte; + } + } + } +} diff --git a/src/CyberDot.Encoding.Base84/README.md b/src/CyberDot.Encoding.Base84/README.md new file mode 100644 index 0000000..fd050aa --- /dev/null +++ b/src/CyberDot.Encoding.Base84/README.md @@ -0,0 +1,44 @@ +# CyberDot.Encoding.Base84 + +An implementation of base84 encoding in C#, using: https://github.com/jedisct1/zig-base84 as a reference. + +Base84 is a binary-to-text encoding that uses 84 ASCII characters, all safe for file names +on macOS, Linux and Windows. It works like Base91, and its output is about 25% larger than +the input, against 33% for Base64. + +## Usage + +```csharp +using CyberDot.Encoding.Base84; + +var bytes = System.Text.Encoding.UTF8.GetBytes("hello world"); +var encoded = Base84.Encode(bytes); + +var decoded = Base84.Decode(encoded); // Output: hello world +``` + +### Streaming + +`ToBase84Transform`/`FromBase84Transform` implement `ICryptoTransform`, so they plug +into `CryptoStream` for streaming encode/decode without buffering the whole payload in +memory. They default to UTF-8, or accept any `Encoding` via the constructor. + +```csharp +using System.Security.Cryptography; + +using var output = new MemoryStream(); +using (var cryptoStream = new CryptoStream(output, new ToBase84Transform(), CryptoStreamMode.Write)) +{ + sourceStream.CopyTo(cryptoStream); +} + +using var decoded = new MemoryStream(); +using (var cryptoStream = new CryptoStream(encodedStream, new FromBase84Transform(), CryptoStreamMode.Read)) +{ + cryptoStream.CopyTo(decoded); +} +``` + +## License + +The MIT License (MIT) diff --git a/src/CyberDot.Encoding.Base84/ToBase84Transform.cs b/src/CyberDot.Encoding.Base84/ToBase84Transform.cs new file mode 100644 index 0000000..4efa7f0 --- /dev/null +++ b/src/CyberDot.Encoding.Base84/ToBase84Transform.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using static CyberDot.Encoding.Base84.Constants; + +namespace CyberDot.Encoding.Base84 +{ + /// + /// Streams raw bytes into their Base84 representation, serialised using whichever + /// is supplied to the constructor (UTF-8 by default). Since the + /// Base84 alphabet is pure ASCII, the resulting text is equally valid under any of + /// them. Intended for use with . + /// + public class ToBase84Transform : ICryptoTransform + { + private readonly System.Text.Encoding encoding; + + // Bit accumulator state, carried across TransformBlock calls: a group of five + // characters holds 31 or 32 input bits, so almost every input byte leaves a + // partial group pending. + private ulong acc; + private int numBits; + + public ToBase84Transform(System.Text.Encoding encoding = null) + { + this.encoding = encoding ?? new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + OutputBlockSize = this.encoding.GetMaxByteCount(GroupChars); + } + + public bool CanReuseTransform => true; + public bool CanTransformMultipleBlocks => true; + public int InputBlockSize => 1; + public int OutputBlockSize { get; } + + public void Dispose() { } + + public int TransformBlock(byte[] inputBuffer, int inputOffset, int inputCount, byte[] outputBuffer, int outputOffset) + { + var output = ProcessBytes(inputBuffer, inputOffset, inputCount, isFinal: false); + Array.Copy(output, 0, outputBuffer, outputOffset, output.Length); + return output.Length; + } + + public byte[] TransformFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount) + { + var result = ProcessBytes(inputBuffer, inputOffset, inputCount, isFinal: true); + acc = 0; + numBits = 0; + return result; + } + + private byte[] ProcessBytes(byte[] inputBuffer, int inputOffset, int inputCount, bool isFinal) + { + if (inputBuffer == null) + { + throw new ArgumentNullException(nameof(inputBuffer)); + } + + if (inputOffset < 0 || inputCount < 0 || inputOffset + inputCount > inputBuffer.Length) + { + throw new ArgumentOutOfRangeException(nameof(inputCount)); + } + + var output = new List(inputCount * 2); + var end = inputOffset + inputCount; + + for (var i = inputOffset; i < end; i++) + { + acc |= (ulong)inputBuffer[i] << numBits; + numBits += BitsPerByte; + + if (numBits > GroupBits) + { + var low = (uint)acc; + var bits = Base84.GroupBitsOf(low); + var v = bits > GroupBits ? low : low & GroupMask; + acc >>= bits; + numBits -= bits; + WriteDigits(v, GroupChars, output); + } + } + + if (isFinal && numBits > 0) + { + var tail = (uint)acc; + var chars = Base84.FitsOneChar(numBits, tail) ? 1 : TailChars[numBits]; + WriteDigits(tail, chars, output); + } + + return output.ToArray(); + } + + private void WriteDigits(uint value, int count, List output) + { + var chars = new char[count]; + for (var i = 0; i < count; i++) + { + chars[i] = Base84.Alphabet[value % Radix]; + value /= Radix; + } + + output.AddRange(encoding.GetBytes(chars)); + } + } +} From b350c479553fc83b20b8e87747ee8b501ae46410 Mon Sep 17 00:00:00 2001 From: cyberdot Date: Tue, 15 Sep 2026 17:48:45 +0100 Subject: [PATCH 2/2] Build/deploy changes for Base84 encoding implementation --- .github/workflows/ci.yml | 3 ++- .github/workflows/release-base84.yml | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release-base84.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cc7aa..4dc53f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,7 @@ jobs: run: | for project in \ src/CyberDot.Encoding.Base65536/CyberDot.Encoding.Base65536.csproj \ - src/CyberDot.Encoding.Base2048/CyberDot.Encoding.Base2048.csproj; do + src/CyberDot.Encoding.Base2048/CyberDot.Encoding.Base2048.csproj \ + src/CyberDot.Encoding.Base84/CyberDot.Encoding.Base84.csproj; do dotnet pack "$project" --configuration Debug --no-build --output "${{ runner.temp }}/nupkg-verify" done diff --git a/.github/workflows/release-base84.yml b/.github/workflows/release-base84.yml new file mode 100644 index 0000000..4eafc99 --- /dev/null +++ b/.github/workflows/release-base84.yml @@ -0,0 +1,22 @@ +name: Release CyberDot.Encoding.Base84 + +# Manually-triggered release for the Base84 package. See _release-package.yml for +# what this actually does. + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version override (e.g. 1.2.0). Leave empty to use the version in the csproj.' + required: false + type: string + +jobs: + release: + permissions: + contents: write + uses: ./.github/workflows/_release-package.yml + with: + project-path: src/CyberDot.Encoding.Base84/CyberDot.Encoding.Base84.csproj + tag-prefix: base84 + version: ${{ inputs.version }}