Transition to the new Base16/Base64 packages

cabal.project

@@ -100,7 +100,9 @@ source-repository-package
 source-repository-package
     type: git
     location: https://github.com/kda-community/pact-5
-    tag: 72f427605406df61be8284091922f1fe1af7541b
+    -- For now, we still use an outdated pin, that still compiles with GHC 9.10
+    -- TO be changed soon
+    tag: 57aa03024a88f3206a31d3d8e7876a5f0983ba4a
 
 source-repository-package
     type: git
@@ -140,14 +142,12 @@ source-repository-package
     tag: c234b19732a0576067305b001c02e783e19e0f2b
     --sha256: 16x2awn8v7dhdbz66a170xbmdj0pyxn1xwinqmsxhlq3qs35ncfj
 
--- Required for non-canonical decode in base64-bytestring (remove after 2.20 fork)
-source-repository-package
-    type: git
-    location: https://github.com/kda-community/base64-bytestring-kadena
-    tag: 90247042ab3b8662809210af2a78e6dee0f9b4ac
-    --sha256: 0dqsrjxm0cm35xcihm49dhwdvmz79vsv4sd5qs2izc4sbnd0d8n6
-
 -- -------------------------------------------------------------------------- --
+-- Make sure we are using the SIMD enabled version of Base16 / Base64
+constraints:
+    base64 +simd
+   ,base16 +simd
+----------------------------------------------------------------------------------
 -- Relaxed Bounds
 
 -- GHC-9:

cabal.project.freeze

@@ -48,12 +48,12 @@ constraints: any.Cabal ==3.12.1.0 || ==3.14.2.0,
              any.base-orphans ==0.9.4,
              any.base-unicode-symbols ==0.2.4.2,
              base-unicode-symbols +base-4-8 -old-base,
-             any.base16 ==1.0,
+             any.base16 ==2.0,
+             base16 +simd,
              any.base16-bytestring ==1.0.2.0,
-             any.base64 ==2.0,
-             base64 -simd,
+             any.base64 ==2.1,
+             base64 +simd,
              any.base64-bytestring ==1.2.1.0,
-             any.base64-bytestring-kadena ==0.1,
              any.basement ==0.0.16,
              any.bifunctors ==5.6.3,
              bifunctors +tagged,
@@ -349,7 +349,7 @@ constraints: any.Cabal ==3.12.1.0 || ==3.14.2.0,
              any.regex ==1.1.0.2,
              any.regex-base ==0.94.0.3,
              any.regex-pcre-builtin ==0.95.2.3.8.44,
-             any.regex-tdfa ==1.3.2.6,
+             any.regex-tdfa ==1.3.2.7,
              regex-tdfa +doctest -force-o2,
              any.resource-pool ==0.5.1.0,
              any.resourcet ==1.3.0,
@@ -416,8 +416,8 @@ constraints: any.Cabal ==3.12.1.0 || ==3.14.2.0,
              any.tagged ==0.8.11,
              tagged +deepseq +template-haskell,
              any.tagsoup ==0.14.8,
-             any.tasty ==1.5.4,
-             tasty +unix,
+             any.tasty ==1.5.5,
+             tasty +terminal-size +unix,
              any.tasty-golden ==2.3.6,
              tasty-golden -build-example,
              any.tasty-hedgehog ==1.4.0.2,
@@ -534,4 +534,4 @@ constraints: any.Cabal ==3.12.1.0 || ==3.14.2.0,
              zip-archive -executable,
              any.zlib ==0.7.1.1,
              zlib -bundled-c-zlib +non-blocking-ffi +pkg-config
-index-state: hackage.haskell.org 2026-10-18T08:35:38Z
+index-state: hackage.haskell.org 2026-10-21T06:37:23Z

chainweb.cabal

@@ -379,8 +379,8 @@ library
         , async >= 2.2
         , attoparsec >= 0.13
         , base >= 4.12 && < 5
-        , base16-bytestring >= 0.1
-        , base64-bytestring-kadena == 0.1
+        , base16 >= 2.0
+        , base64 >= 2.1
         , binary >= 0.8
         , bytestring >= 0.10.12
         , case-insensitive >= 1.2
@@ -538,7 +538,7 @@ library chainweb-test-utils
         , aeson >= 2.2
         , async >= 2.2
         , base >= 4.12 && < 5
-        , base16-bytestring >= 0.1
+        , base16 >= 2.0
         , bytestring >= 0.10.12
         , case-insensitive >= 1.2
         , chainweb-storage >= 0.1
@@ -696,8 +696,7 @@ test-suite chainweb-tests
         , aeson >= 2.2
         , async >= 2.2
         , base >= 4.12 && < 5
-        , base16-bytestring >= 0.1
-        , base64-bytestring-kadena == 0.1
+        , base16 >= 2.0
         , byteslice >= 0.2.12
         , bytesmith >= 0.3.10
         , bytestring >= 0.10.12

src/Chainweb/BlockHeaderDB/RestAPI/Server.hs

@@ -39,6 +39,7 @@ import Control.Monad.IO.Class
 import Data.Aeson
 import Data.Binary.Builder (fromByteString, fromLazyByteString)
 import qualified Data.ByteString as BS
+import qualified Data.Base16.Types as B16
 import qualified Data.ByteString.Base16 as B16
 import Data.ByteString.Short (fromShort)
 import Data.Foldable
@@ -47,7 +48,6 @@ import Data.Functor.Of
 import Data.IORef
 import qualified Data.Map.Strict as Map
 import Data.Proxy
-import Data.Text.Encoding (decodeUtf8)
 import Numeric.Natural(Natural)
 
 import Network.Wai.EventSource (ServerEvent(..), eventSourceAppIO)
@@ -423,7 +423,7 @@ blockStreamHandler db withPayloads = Tagged $ \req resp -> do
             , _huPayloadWithOutputs =
                 x <$ guard withPayloads
             , _huTxCount = length $ _payloadWithOutputsTransactions x
-            , _huPowHash = decodeUtf8 . B16.encode . BS.reverse . fromShort . powHashBytes $ view blockPow bh
+            , _huPowHash = B16.extractBase16 . B16.encodeBase16 . BS.reverse . fromShort . powHashBytes $ view blockPow bh
             , _huTarget = showTargetHex $ view blockTarget bh
             }
 

src/Chainweb/Mempool/Mempool.hs

@@ -108,6 +108,7 @@ import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Short as SB
 import Data.Decimal (Decimal, DecimalRaw(..))
 import Data.Foldable (traverse_)
+import Data.Bifunctor (first)
 import Data.Hashable (Hashable(hashWithSalt))
 import Data.HashSet (HashSet)
 import qualified Data.HashSet as HashSet
@@ -750,7 +751,7 @@ mockCodec = Codec mockEncode mockDecode
 
 mockEncode :: MockTx -> ByteString
 mockEncode (MockTx nonce (GasPrice (ParsedDecimal price)) limit meta) =
-  B64.encode $
+  encodeB64UrlNoPadding $
   runPutS $ do
     putWord64le $ fromIntegral nonce
     putDecimal price
@@ -793,7 +794,7 @@ getDecimal = do
 
 mockDecode :: ByteString -> Either String MockTx
 mockDecode s = do
-    s' <- B64.decode s
+    s' <- first T.unpack $ B64.decodeBase64UnpaddedUntyped s
     runGetEitherS (MockTx <$> getI64 <*> getPrice <*> getGL <*> getMeta) s'
   where
     getPrice = GasPrice . ParsedDecimal <$> getDecimal

src/Chainweb/Pact/Backend/PactState/EmbeddedSnapshot.hs

@@ -12,7 +12,8 @@ import Chainweb.BlockHeader (BlockHeader)
 import Chainweb.Pact.Backend.PactState.GrandHash.Algorithm (ChainGrandHash)
 import Data.Aeson qualified as A
 import Data.ByteString (ByteString)
-import Data.ByteString.Base16 qualified as Base16
+import Data.Base16.Types qualified as B16
+import Data.ByteString.Base16 qualified as B16
 import Data.Text (Text)
 import Data.Text.Encoding qualified as Text
 
@@ -28,6 +29,4 @@ unsafeDecodeBlockHeader t = case A.decodeStrict (Text.encodeUtf8 t) of
   Nothing -> error "unsafeDecodeBlockHeader: invalid BlockHeader JSON"
 
 unsafeBase16Decode :: Text -> ByteString
-unsafeBase16Decode t = case Base16.decode (Text.encodeUtf8 t) of
-  Right a -> a
-  Left err -> error ("unsafeBase16Decode failed: " ++ show err)
+unsafeBase16Decode = B16.decodeBase16' . B16.assertBase16

src/Chainweb/Pact/Backend/PactState/GrandHash/Algorithm.hs

@@ -28,7 +28,9 @@ import Data.Bifunctor (first)
 import Data.ByteArray qualified as Memory
 import Data.ByteString (ByteString)
 import Data.ByteString qualified as BS
-import Data.ByteString.Base16 qualified as Base16
+import Data.Coerce (coerce)
+import Data.Base16.Types qualified as B16
+import Data.ByteString.Base16 qualified as B16
 import Data.ByteString.Builder qualified as BB
 import Data.ByteString.Lazy qualified as BL
 import Data.Int (Int64)
@@ -58,7 +60,7 @@ newtype TableHash = TableHash { getTableHash :: ByteString }
 newtype HexBytes = HexBytes ByteString
 
 instance Show HexBytes where
-  show (HexBytes b) = Text.unpack (Text.decodeUtf8 (Base16.encode b))
+  show = Text.unpack . B16.extractBase16 . B16.encodeBase16 . coerce
 
 computeGrandHash :: forall m r. (Monad m)
   => Stream (Of (Text, Map ByteString PactRowContents)) m r

src/Chainweb/Pact/Backend/PactState/GrandHash/Utils.hs

@@ -46,13 +46,13 @@ import Control.Exception (bracket)
 import Control.Lens ((^?!), ix, view)
 import Control.Monad (forM, when)
 import Data.ByteString (ByteString)
-import Data.ByteString.Base16 qualified as Base16
+import Data.Base16.Types qualified as B16
+import Data.ByteString.Base16 qualified as B16
 import Data.HashMap.Strict (HashMap)
 import Data.HashMap.Strict qualified as HM
 import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import Data.Text qualified as Text
-import Data.Text.Encoding qualified as Text
 import Options.Applicative qualified as O
 import System.Directory (doesFileExist)
 import System.Exit (exitFailure)
@@ -216,7 +216,7 @@ withConnections logger pactDir cids f = do
     closeConnections = mapM_ stopSqliteDb
 
 hex :: ByteString -> Text
-hex = Text.decodeUtf8 . Base16.encode
+hex = B16.extractBase16 . B16.encodeBase16
 
 cwvParser :: O.Parser ChainwebVersion
 cwvParser = fmap (lookupVersionByName . fromMaybe (error "ChainwebVersion parse failed") . fromText)

src/Chainweb/Pact4/SPV.hs

@@ -196,33 +196,34 @@ decodeB64UrlNoPaddingTextWithFixedErrorMessage errorMessageType msg =
   fromEitherM
   . first (\e -> Base64DecodeException $ base64ErrorMessage e)
   . (if errorMessageType == Legacy then patchNonCanonical else id)
-  . B64U.decode
+  . B64U.decodeBase64Untyped
   . Text.encodeUtf8
   . (if errorMessageType == Legacy then pad else id)
   $ msg
   where
     pad t = let s = Text.length t `mod` 4 in t <> Text.replicate ((4 - s) `mod` 4) "="
     base64ErrorMessage m = case errorMessageType of
-      Legacy -> base64DowngradeErrorMessage (Text.pack m)
+      Legacy -> base64DowngradeErrorMessage m
       Simplified -> "could not base64-decode message"
     patchNonCanonical decodeResult = case decodeResult of
       Right bs -> Right bs
-      Left e | "non-canonical" `Text.isInfixOf` Text.pack e ->
-               (B64U.decodeNonCanonical (Text.encodeUtf8 msg))
+      Left e | "non-canonical" `Text.isInfixOf` e ->
+               Right $ B64U.decodeBase64Lenient $ Text.encodeUtf8 msg
       Left e -> Left e
 {-# INLINE decodeB64UrlNoPaddingTextWithFixedErrorMessage #-}
 
 
 
--- | Converts the error message format of base64-bytestring-1.2
+-- | Converts the error message format of base64-bytestring-1.2 or base64 2.0
 --   into that of base64-bytestring-0.1, for the error messages
 --   that have made it onto the chain.
---   This allows us to upgrade to base64-bytestring-1.2 without
+--   This allows us to upgrade to base64 without
 --   breaking compatibility.
 base64DowngradeErrorMessage :: Text -> Text
 base64DowngradeErrorMessage msg = case msg of
   "Base64-encoded bytestring has invalid size" ->
        "invalid base64 encoding near offset 0"
+-- Note : to be confirmed, but the two next shims shouldn't be necessary with base64 2.0
   (Text.stripPrefix "invalid character at offset: " -> Just suffix) ->
     Text.pack $ "invalid base64 encoding near offset " ++ show (adjustedOffset suffix)
   (Text.stripPrefix "invalid padding at offset: " -> Just suffix) ->

src/Chainweb/SPV/EventProof.hs

@@ -109,6 +109,7 @@ import Crypto.Hash.Algorithms
 import Data.Aeson
 import qualified Data.ByteArray as BA
 import qualified Data.ByteString as B
+import qualified Data.Base16.Types as B16
 import qualified Data.ByteString.Base16 as B16
 import qualified Data.ByteString.Short as BS
 import Data.Decimal
@@ -260,7 +261,7 @@ getInt256Be = do
 
 int256Hex :: Int256 -> B.ByteString
 int256Hex x@(Int256 i)
-  | i >= 0 = "0x" <> B16.encode (runPutS $ putInt256Be x)
+  | i >= 0 = "0x" <> (B16.extractBase16 $ B16.encodeBase16' $ runPutS $ putInt256Be x)
   | otherwise = "-" <> int256Hex (Int256 (-i))
 
 -- -------------------------------------------------------------------------- --

src/Chainweb/Utils.hs

@@ -109,6 +109,7 @@ module Chainweb.Utils
 -- ** Base64
 , encodeB64Text
 , decodeB64Text
+, encodeB64Url
 , encodeB64UrlText
 , decodeB64UrlText
 , encodeB64UrlNoPadding
@@ -266,10 +267,10 @@ import Data.Bifunctor
 import Data.Bool (bool)
 import Data.ByteString (ByteString)
 import Data.ByteString qualified as B
+import Data.Base64.Types qualified as B64
 import Data.ByteString.Base64 qualified as B64
 import Data.ByteString.Base64.URL qualified as B64U
 import Data.ByteString.Builder qualified as BB
-import Data.ByteString.Char8 qualified as B8
 import Data.ByteString.Lazy qualified as BL
 import Data.Csv qualified as CSV
 import Data.Decimal
@@ -662,15 +663,15 @@ strip0x t = case T.stripPrefix "0x" t of
 --
 decodeB64Text :: MonadThrow m => T.Text -> m B.ByteString
 decodeB64Text = fromEitherM
-    . first (Base64DecodeException . T.pack)
-    . B64.decode
+    . first Base64DecodeException
+    . B64.decodeBase64Untyped
     . T.encodeUtf8
 {-# INLINE decodeB64Text #-}
 
 -- | Encode a binary value to a textual base64 representation.
 --
 encodeB64Text :: B.ByteString -> T.Text
-encodeB64Text = T.decodeUtf8 . B64.encode
+encodeB64Text = B64.extractBase64 . B64.encodeBase64
 {-# INLINE encodeB64Text #-}
 
 -- | Decode a binary value from a textual base64-url representation. A
@@ -679,50 +680,53 @@ encodeB64Text = T.decodeUtf8 . B64.encode
 --
 decodeB64UrlText :: MonadThrow m => T.Text -> m B.ByteString
 decodeB64UrlText = fromEitherM
-    . first (Base64DecodeException . T.pack)
-    . B64U.decode
+    . first Base64DecodeException
+    . B64U.decodeBase64Untyped
     . T.encodeUtf8
 {-# INLINE decodeB64UrlText #-}
 
+
+-- | Encode a binary value to a textual base64-url witg Padding
+encodeB64Url :: B.ByteString -> B.ByteString
+encodeB64Url = B64.extractBase64 . B64U.encodeBase64'
+{-# INLINE encodeB64UrlText #-}
+
 -- | Encode a binary value to a textual base64-url representation.
 --
 encodeB64UrlText :: B.ByteString -> T.Text
-encodeB64UrlText = T.decodeUtf8 . B64U.encode
-{-# INLINE encodeB64UrlText #-}
+encodeB64UrlText = B64.extractBase64 . B64U.encodeBase64
+{-# INLINE encodeB64Url #-}
 
 -- | Decode a binary value from a textual base64-url without padding
 -- representation. A 'Base64DecodeException' is thrown if the input is not a
 -- valid base64-url without padding encoding.
 --
 decodeB64UrlNoPaddingText :: MonadThrow m => T.Text -> m B.ByteString
 decodeB64UrlNoPaddingText = fromEitherM
-    . first (Base64DecodeException . T.pack)
-    . B64U.decode
+    . first Base64DecodeException
+    . B64U.decodeBase64UnpaddedUntyped
     . T.encodeUtf8
-    . pad
-  where
-    pad t = let s = T.length t `mod` 4 in t <> T.replicate ((4 - s) `mod` 4) "="
 {-# INLINE decodeB64UrlNoPaddingText #-}
 
 -- | Encode a binary value to a textual base64-url without padding
 -- representation.
 --
 encodeB64UrlNoPaddingText :: B.ByteString -> T.Text
-encodeB64UrlNoPaddingText = T.dropWhileEnd (== '=') . T.decodeUtf8 . B64U.encode
+encodeB64UrlNoPaddingText = B64.extractBase64 . B64U.encodeBase64Unpadded
 {-# INLINE encodeB64UrlNoPaddingText #-}
 
 -- | Encode a binary value to a textual base64-url without padding
 -- representation.
 --
 encodeB64UrlNoPadding :: B.ByteString -> B.ByteString
-encodeB64UrlNoPadding = B8.dropWhileEnd (== '=') . B64U.encode
+encodeB64UrlNoPadding = B64.extractBase64 . B64U.encodeBase64Unpadded'
 {-# INLINE encodeB64UrlNoPadding #-}
 
 -- | Encode a binary value to a base64-url (without padding) JSON encoding.
 --
 b64UrlNoPaddingTextEncoding :: B.ByteString -> Encoding
 b64UrlNoPaddingTextEncoding t =
-    Aeson.unsafeToEncoding $ BB.char8 '\"' <> BB.byteString (B8.dropWhileEnd (== '=') $ B64U.encode t) <> BB.char8 '\"'
+    Aeson.unsafeToEncoding $ BB.char8 '\"' <> (BB.byteString . encodeB64UrlNoPadding) t <> BB.char8 '\"'
 
 -- -------------------------------------------------------------------------- --
 -- ** JSON

src/Chainweb/VerifierPlugin/Hyperlane/Utils.hs

@@ -24,6 +24,7 @@ module Chainweb.VerifierPlugin.Hyperlane.Utils
 import Control.Lens hiding (index)
 import Control.Monad.Catch
 
+import Data.Bifunctor (first)
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Base16 as B16
 import qualified Data.ByteString.Short as BS
@@ -78,7 +79,7 @@ encodeHex = ((<>) "0x") . Text.decodeUtf8 . B.toStrict . Builder.toLazyByteStrin
 
 decodeHex :: Text -> Either String B.ByteString
 decodeHex s
-  | Just h <- Text.stripPrefix "0x" s = B16.decode $ Text.encodeUtf8 h
+  | Just h <- Text.stripPrefix "0x" s = first (const "Base16 Decoding Error") $ B16.decodeBase16Untyped $ Text.encodeUtf8 h
   | otherwise = Left "decodeHex: does not start with 0x"
 
 decodeHexUnsafe :: Text -> B.ByteString