exo2: Implement (untested) SmcDecryptDeviceUniqueData

libexosphere/source/crypto/crypto_aes_impl_security_engine.cpp

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <exosphere.hpp>
+
+namespace ams::crypto::impl {
+
+    namespace {
+
+        constexpr bool IsSupportedKeySize(size_t size) {
+            return size == 16 || size == 24 || size == 32;
+        }
+
+    }
+
+    template<size_t KeySize>
+    AesImpl<KeySize>::~AesImpl() {
+        ClearMemory(this, sizeof(*this));
+    }
+
+    template<size_t KeySize>
+    void AesImpl<KeySize>::Initialize(const void *key, size_t key_size, bool is_encrypt) {
+        static_assert(IsSupportedKeySize(KeySize));
+
+        /* Set the security engine keyslot. */
+        this->slot = *static_cast<const int *>(key);
+    }
+
+    template<size_t KeySize>
+    void AesImpl<KeySize>::EncryptBlock(void *dst, size_t dst_size, const void *src, size_t src_size) const {
+        static_assert(IsSupportedKeySize(KeySize));
+        AMS_ASSERT(src_size >= BlockSize);
+        AMS_ASSERT(dst_size >= BlockSize);
+
+        if constexpr (KeySize == 16) {
+            /* Aes 128. */
+            se::EncryptAes128(dst, dst_size, this->slot, src, src_size);
+        } else if constexpr (KeySize == 24) {
+            /* Aes 192. */
+            /* TODO: se::EncryptAes192(dst, dst_size, this->slot, src, src_size); */
+        } else if constexpr (KeySize == 32) {
+            /* Aes 256. */
+            /* TODO: se::EncryptAes256(dst, dst_size, this->slot, src, src_size); */
+        } else {
+            /* Invalid key size. */
+            static_assert(!std::is_same<AesImpl<KeySize>, AesImpl<KeySize>>::value);
+        }
+    }
+
+    template<size_t KeySize>
+    void AesImpl<KeySize>::DecryptBlock(void *dst, size_t dst_size, const void *src, size_t src_size) const {
+        static_assert(IsSupportedKeySize(KeySize));
+        AMS_ASSERT(src_size >= BlockSize);
+        AMS_ASSERT(dst_size >= BlockSize);
+
+        if constexpr (KeySize == 16) {
+            /* Aes 128. */
+            se::DecryptAes128(dst, dst_size, this->slot, src, src_size);
+        } else if constexpr (KeySize == 24) {
+            /* Aes 192. */
+            /* TODO: se::DecryptAes192(dst, dst_size, this->slot, src, src_size); */
+        } else if constexpr (KeySize == 32) {
+            /* Aes 256. */
+            /* TODO: se::DecryptAes256(dst, dst_size, this->slot, src, src_size); */
+        } else {
+            /* Invalid key size. */
+            static_assert(!std::is_same<AesImpl<KeySize>, AesImpl<KeySize>>::value);
+        }
+    }
+
+
+    /* Explicitly instantiate the three supported key sizes. */
+    template class AesImpl<16>;
+    template class AesImpl<24>;
+    template class AesImpl<32>;
+
+}

libvapours/include/vapours/crypto.hpp

@@ -25,6 +25,7 @@
 #include <vapours/crypto/crypto_aes_decryptor.hpp>
 #include <vapours/crypto/crypto_aes_ctr_encryptor_decryptor.hpp>
 #include <vapours/crypto/crypto_aes_xts_encryptor_decryptor.hpp>
+#include <vapours/crypto/crypto_aes_gcm_encryptor.hpp>
 #include <vapours/crypto/crypto_rsa_pss_sha256_verifier.hpp>
 #include <vapours/crypto/crypto_rsa_oaep_sha256_decoder.hpp>
 #include <vapours/crypto/crypto_rsa_oaep_sha256_decryptor.hpp>

libvapours/include/vapours/crypto/crypto_aes_gcm_encryptor.hpp

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <vapours/common.hpp>
+#include <vapours/assert.hpp>
+#include <vapours/util.hpp>
+#include <vapours/crypto/crypto_aes_encryptor.hpp>
+#include <vapours/crypto/crypto_gcm_encryptor.hpp>
+
+namespace ams::crypto {
+
+    namespace impl {
+
+        template<typename _AesImpl>
+        class AesGcmEncryptor {
+            NON_COPYABLE(AesGcmEncryptor);
+            NON_MOVEABLE(AesGcmEncryptor);
+            private:
+                using AesImpl = _AesImpl;
+                using GcmImpl = GcmEncryptor<AesImpl>;
+            public:
+                static constexpr size_t KeySize   = AesImpl::KeySize;
+                static constexpr size_t BlockSize = AesImpl::BlockSize;
+                static constexpr size_t MacSize   = AesImpl::BlockSize;
+            private:
+                AesImpl aes_impl;
+                GcmImpl gcm_impl;
+            public:
+                AesGcmEncryptor() { /* ... */ }
+
+                void Initialize(const void *key, size_t key_size, const void *iv, size_t iv_size) {
+                    this->aes_impl.Initialize(key, key_size);
+                    this->gcm_impl.Initialize(std::addressof(this->aes_impl), iv, iv_size);
+                }
+
+                void Reset(const void *iv, size_t iv_size) {
+                    this->gcm_impl.Reset(iv, iv_size);
+                }
+
+                size_t Update(void *dst, size_t dst_size, const void *src, size_t src_size) {
+                    return this->gcm_impl.Update(dst, dst_size, src, src_size);
+                }
+
+                void UpdateAad(const void *aad, size_t aad_size) {
+                    return this->gcm_impl.UpdateAad(aad, aad_size);
+                }
+
+                void GetMac(void *dst, size_t dst_size) {
+                    return this->gcm_impl.GetMac(dst, dst_size);
+                }
+        };
+
+    }
+
+    using Aes128GcmEncryptor = impl::AesGcmEncryptor<AesEncryptor128>;
+    /* TODO: Validate AAD/GMAC is same for non-128 bit key using Aes192GcmEncryptor = impl::AesGcmEncryptor<AesEncryptor192>; */
+    /* TODO: Validate AAD/GMAC is same for non-128 bit key using Aes256GcmEncryptor = impl::AesGcmEncryptor<AesEncryptor256>; */
+
+}

libvapours/include/vapours/crypto/crypto_gcm_encryptor.hpp

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <vapours/common.hpp>
+#include <vapours/assert.hpp>
+#include <vapours/util.hpp>
+#include <vapours/crypto/impl/crypto_gcm_mode_impl.hpp>
+
+namespace ams::crypto {
+
+    /* TODO: C++20 BlockCipher concept */
+
+    template<typename BlockCipher>
+    class GcmEncryptor {
+        NON_COPYABLE(GcmEncryptor);
+        NON_MOVEABLE(GcmEncryptor);
+        private:
+            using Impl = impl::GcmModeImpl<BlockCipher>;
+        public:
+            static constexpr size_t KeySize   = Impl::KeySize;
+            static constexpr size_t BlockSize = Impl::BlockSize;
+            static constexpr size_t MacSize   = Impl::MacSize;
+        private:
+            Impl impl;
+        public:
+            GcmEncryptor() { /* ... */ }
+
+            void Initialize(const BlockCipher *cipher, const void *iv, size_t iv_size) {
+                this->impl.Initialize(cipher);
+                this->impl.Reset(iv, iv_size);
+            }
+
+            void Reset(const void *iv, size_t iv_size) {
+                this->impl.Reset(iv, iv_size);
+            }
+
+            size_t Update(void *dst, size_t dst_size, const void *src, size_t src_size) {
+                return this->impl.Update(dst, dst_size, src, src_size);
+            }
+
+            void UpdateAad(const void *aad, size_t aad_size) {
+                return this->impl.UpdateAad(aad, aad_size);
+            }
+
+            void GetMac(void *dst, size_t dst_size) {
+                return this->impl.GetMac(dst, dst_size);
+            }
+    };
+
+}

libvapours/include/vapours/crypto/impl/crypto_aes_impl.hpp

@@ -30,16 +30,24 @@ namespace ams::crypto::impl {
             static constexpr s32    RoundCount   = (KeySize / 4) + 6;
             static constexpr size_t RoundKeySize = BlockSize * (RoundCount + 1);
         private:
+        #ifdef ATMOSPHERE_IS_EXOSPHERE
+            int slot;
+        #endif
+        #ifdef ATMOSPHERE_IS_STRATOSPHERE
             u32 round_keys[RoundKeySize / sizeof(u32)];
+        #endif
         public:
             ~AesImpl();
 
             void Initialize(const void *key, size_t key_size, bool is_encrypt);
             void EncryptBlock(void *dst, size_t dst_size, const void *src, size_t src_size) const;
             void DecryptBlock(void *dst, size_t dst_size, const void *src, size_t src_size) const;
+
+        #ifdef ATMOSPHERE_IS_STRATOSPHERE
             const u8 *GetRoundKey() const {
                 return reinterpret_cast<const u8 *>(this->round_keys);
             }
+        #endif
     };
 
     /* static_assert(HashFunction<Sha1Impl>); */

libvapours/include/vapours/crypto/impl/crypto_gcm_mode_impl.hpp

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <vapours/common.hpp>
+#include <vapours/assert.hpp>
+#include <vapours/util.hpp>
+#include <vapours/crypto/crypto_memory_clear.hpp>
+#include <vapours/crypto/crypto_aes_encryptor.hpp>
+
+namespace ams::crypto::impl {
+
+    template<typename BlockCipher>
+    class GcmModeImpl {
+        NON_COPYABLE(GcmModeImpl);
+        NON_MOVEABLE(GcmModeImpl);
+        public:
+            static constexpr size_t KeySize   = BlockCipher::KeySize;
+            static constexpr size_t BlockSize = BlockCipher::BlockSize;
+            static constexpr size_t MacSize   = BlockCipher::BlockSize;
+        private:
+            enum State {
+                State_None,
+                State_Initialized,
+                State_ProcessingAad,
+                State_Encrypting,
+                State_Decrypting,
+                State_Done,
+            };
+
+            struct Block128 {
+                u64 hi;
+                u64 lo;
+
+                ALWAYS_INLINE void Clear() {
+                    this->hi = 0;
+                    this->lo = 0;
+                }
+            };
+            static_assert(util::is_pod<Block128>::value);
+            static_assert(sizeof(Block128) == 0x10);
+
+            union Block {
+                Block128 block_128;
+                u32      block_32[4];
+                u8       block_8[16];
+            };
+            static_assert(util::is_pod<Block>::value);
+            static_assert(sizeof(Block) == 0x10);
+
+            using CipherFunction = void (*)(void *dst_block, const void *src_block, const void *ctx);
+        private:
+            State state;
+            const BlockCipher *block_cipher;
+            CipherFunction cipher_func;
+            u8 pad[sizeof(u64)];
+            Block block_x;
+            Block block_y;
+            Block block_ek;
+            Block block_ek0;
+            Block block_tmp;
+            size_t aad_size;
+            size_t msg_size;
+            u32    aad_remaining;
+            u32    msg_remaining;
+            u32    counter;
+            Block  h_mult_blocks[16];
+        public:
+            GcmModeImpl() : state(State_None) { /* ... */ }
+
+            ~GcmModeImpl() {
+                ClearMemory(this, sizeof(*this));
+            }
+
+            void Initialize(const BlockCipher *block_cipher);
+
+            void Reset(const void *iv, size_t iv_size);
+
+            void   UpdateAad(const void *aad, size_t aad_size);
+            size_t UpdateEncrypt(void *dst, size_t dst_size, const void *src, size_t src_size);
+            size_t UpdateDecrypt(void *dst, size_t dst_size, const void *src, size_t src_size);
+
+            void GetMac(void *dst, size_t dst_size);
+        private:
+            static void ProcessBlock(void *dst_block, const void *src_block, const void *ctx) {
+                static_cast<const BlockCipher *>(ctx)->EncryptBlock(dst_block, BlockSize, src_block, BlockSize);
+            }
+
+            void InitializeHashKey();
+            void ComputeMac(bool encrypt);
+    };
+
+}

libvapours/include/vapours/util/util_endian.hpp

@@ -43,7 +43,6 @@ namespace ams::util {
                     ((u & (ByteMask << 16)) << 24) |
                     ((u & (ByteMask <<  8)) << 40) |
                     ((u & (ByteMask <<  0)) << 56);
-
         } else if constexpr (std::is_same<U, u32>::value) {
             return  ((u & (ByteMask << 24)) >> 24) |
                     ((u & (ByteMask << 16)) >>  8) |
@@ -79,7 +78,7 @@ namespace ams::util {
     constexpr ALWAYS_INLINE void SwapBytes(T *ptr) {
         using U = typename std::make_unsigned<T>::type;
 
-        *ptr = static_cast<T>(SwapBytes(static_cast<U>(*ptr)));
+        *ptr = static_cast<T>(SwapBytes<U>(static_cast<U>(*ptr)));
     }
 
     template<typename T> requires std::integral<T>
@@ -90,7 +89,7 @@ namespace ams::util {
             return static_cast<T>(static_cast<U>(val));
         } else {
             static_assert(IsLittleEndian());
-            return static_cast<T>(SwapBytes(static_cast<U>(val)));
+            return static_cast<T>(SwapBytes<U>(static_cast<U>(val)));
         }
     }
 
@@ -99,7 +98,7 @@ namespace ams::util {
         using U = typename std::make_unsigned<T>::type;
 
         if constexpr (IsBigEndian()) {
-            return static_cast<T>(SwapBytes(static_cast<U>(val)));
+            return static_cast<T>(SwapBytes<U>(static_cast<U>(val)));
         } else {
             static_assert(IsLittleEndian());
             return static_cast<T>(static_cast<U>(val));
@@ -136,22 +135,22 @@ namespace ams::util {
 
     template<typename T> requires std::integral<T>
     constexpr ALWAYS_INLINE T LoadBigEndian(const T *ptr) {
-        return ConvertToBigEndian(*ptr);
+        return ConvertToBigEndian<T>(*ptr);
     }
 
     template<typename T> requires std::integral<T>
     constexpr ALWAYS_INLINE T LoadLittleEndian(const T *ptr) {
-        return ConvertToLittleEndian(*ptr);
+        return ConvertToLittleEndian<T>(*ptr);
     }
 
     template<typename T> requires std::integral<T>
     constexpr ALWAYS_INLINE void StoreBigEndian(T *ptr, T val) {
-        *ptr = ConvertToBigEndian(val);
+        *ptr = ConvertToBigEndian<T>(val);
     }
 
     template<typename T> requires std::integral<T>
     constexpr ALWAYS_INLINE void StoreLittleEndian(T *ptr, T val) {
-        *ptr = ConvertToLittleEndian(val);
+        *ptr = ConvertToLittleEndian<T>(val);
     }
 
 }

libvapours/source/crypto/impl/crypto_aes_impl.arch.arm64.cpp

@@ -111,6 +111,8 @@ namespace ams::crypto::impl {
 
 #else
 
+    /* NOTE: Exosphere defines this in libexosphere. */
+
     /* TODO: Non-EL0 implementation. */
 
 #endif

libvapours/source/crypto/impl/crypto_gcm_mode_impl.arch.arm64.cpp

@@ -0,0 +1,305 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <vapours.hpp>
+
+#if defined(ATMOSPHERE_IS_STRATOSPHERE)
+
+/* TODO: EL0 implementation. */
+namespace ams::crypto::impl {
+
+}
+
+#else
+
+/* EL1+ implementation. */
+namespace ams::crypto::impl {
+
+    namespace {
+
+        constexpr u64 GetMultiplyFactor(u8 value) {
+            constexpr size_t Shift = BITSIZEOF(u8) - 1;
+            constexpr u8     Mask  = (1u << Shift);
+            return (value & Mask) >> Shift;
+        }
+
+        /* TODO: Big endian support, eventually? */
+        constexpr void GaloisShiftLeft(u64 *block) {
+            /* Shift the block left by one. */
+            block[1] <<= 1;
+            block[1] |= (block[0] & (static_cast<u64>(1) << (BITSIZEOF(u64) - 1))) >> (BITSIZEOF(u64) - 1);
+            block[0] <<= 1;
+        }
+
+        constexpr u8 GaloisShiftRight(u64 *block) {
+            /* Determine the mask to return. */
+            constexpr u8 GaloisFieldMask = 0xE1;
+            const u8 mask = (block[0] & 1) * GaloisFieldMask;
+
+            /* Shift the block right by one. */
+            block[0] >>= 1;
+            block[0] |= (block[1] & 1) << (BITSIZEOF(u64) - 1);
+            block[1] >>= 1;
+
+            /* Return the mask. */
+            return mask;
+        }
+
+        /* Multiply two 128-bit numbers X, Y in the GF(128) Galois Field. */
+        void GaloisFieldMult(void *dst, const void *x, const void *y) {
+            /* Our block size is 16 bytes (for a 128-bit integer). */
+            constexpr size_t BlockSize = 16;
+            constexpr size_t FieldSize = 128;
+
+            /* Declare work blocks for us to store temporary values. */
+            u8 x_block[BlockSize];
+            u8 y_block[BlockSize];
+            u8 out[BlockSize];
+
+            /* Declare 64-bit pointers for our convenience. */
+            u64 *x_64   = static_cast<u64 *>(static_cast<void *>(x_block));
+            u64 *y_64   = static_cast<u64 *>(static_cast<void *>(y_block));
+            u64 *out_64 = static_cast<u64 *>(static_cast<void *>(out));
+
+            /* Initialize our work blocks. */
+            for (size_t i = 0; i < BlockSize; ++i) {
+                x_block[i] = static_cast<const u8 *>(x)[BlockSize - 1 - i];
+                y_block[i] = static_cast<const u8 *>(y)[BlockSize - 1 - i];
+                out[i]     = 0;
+            }
+
+            /* Perform multiplication on each bit in y. */
+            for (size_t i = 0; i < FieldSize; ++i) {
+                /* Get the multiply factor for this bit. */
+                const auto y_mult = GetMultiplyFactor(y_block[BlockSize - 1]);
+
+                /* Multiply x by the factor. */
+                out_64[0] ^= x_64[0] * y_mult;
+                out_64[1] ^= x_64[1] * y_mult;
+
+                /* Shift left y by one. */
+                GaloisShiftLeft(y_64);
+
+                /* Shift right x by one, and mask appropriately. */
+                const u8 x_mask = GaloisShiftRight(x_64);
+                x_block[BlockSize - 1] ^= x_mask;
+            }
+
+            /* Copy out our result. */
+            for (size_t i = 0; i < BlockSize; ++i) {
+                static_cast<u8 *>(dst)[i] = out[BlockSize - 1 - i];
+            }
+        }
+
+    }
+
+    template<class BlockCipher>
+    void GcmModeImpl<BlockCipher>::Initialize(const BlockCipher *block_cipher) {
+        /* Set member variables. */
+        this->block_cipher = block_cipher;
+        this->cipher_func  = std::addressof(GcmModeImpl<BlockCipher>::ProcessBlock);
+
+        /* Pre-calculate values to speed up galois field multiplications later. */
+        this->InitializeHashKey();
+
+        /* Note that we're initialized. */
+        this->state = State_Initialized;
+    }
+
+    template<class BlockCipher>
+    void GcmModeImpl<BlockCipher>::Reset(const void *iv, size_t iv_size) {
+        /* Validate pre-conditions. */
+        AMS_ASSERT(this->state >= State_Initialized);
+
+        /* Reset blocks. */
+        this->block_x.block_128.Clear();
+        this->block_tmp.block_128.Clear();
+
+        /* Clear sizes. */
+        this->aad_size      = 0;
+        this->msg_size      = 0;
+        this->aad_remaining = 0;
+        this->msg_remaining = 0;
+
+        /* Update our state. */
+        this->state = State_ProcessingAad;
+
+        /* Set our iv. */
+        if (iv_size == 12) {
+            /* If our iv is the correct size, simply copy in the iv, and set the magic bit. */
+            std::memcpy(std::addressof(this->block_ek0), iv, iv_size);
+            util::StoreBigEndian(this->block_ek0.block_32 + 3, static_cast<u32>(1));
+        } else {
+            /* Clear our ek0 block. */
+            this->block_ek0.block_128.Clear();
+
+            /* Update using the iv as aad. */
+            this->UpdateAad(iv, iv_size);
+
+            /* Treat the iv as fake msg for the mac that will become our iv. */
+            this->msg_size = this->aad_size;
+            this->aad_size = 0;
+
+            /* Compute a non-final mac. */
+            this->ComputeMac(false);
+
+            /* Set our ek0 block to our calculated mac block. */
+            this->block_ek0 = this->block_x;
+
+            /* Clear our calculated mac block. */
+            this->block_x.block_128.Clear();
+
+            /* Reset our state. */
+            this->msg_size      = 0;
+            this->aad_size      = 0;
+            this->msg_remaining = 0;
+            this->aad_remaining = 0;
+        }
+
+        /* Set the working block to the iv. */
+        this->block_ek = this->block_ek0;
+    }
+
+    template<class BlockCipher>
+    void GcmModeImpl<BlockCipher>::UpdateAad(const void *aad, size_t aad_size) {
+        /* Validate pre-conditions. */
+        AMS_ASSERT(this->state    == State_ProcessingAad);
+        AMS_ASSERT(this->msg_size == 0);
+
+        /* Update our aad size. */
+        this->aad_size += aad_size;
+
+        /* Define a working tracker variable. */
+        const u8 *cur_aad = static_cast<const u8 *>(aad);
+
+        /* Process any leftover aad data from a previous invocation. */
+        if (this->aad_remaining > 0) {
+            while (aad_size > 0) {
+                /* Copy in a byte of the aad to our partial block. */
+                this->block_x.block_8[BlockSize - 1 - this->aad_remaining] ^= *(cur_aad++);
+
+                /* Note that we consumed a byte. */
+                --aad_size;
+
+                /* Increment our partial block size. */
+                this->aad_remaining = (this->aad_remaining + 1) % BlockSize;
+
+                /* If we have a complete block, process it and move onward. */
+                GaloisFieldMult(std::addressof(this->block_x), std::addressof(this->block_x), std::addressof(this->h_mult_blocks[0]));
+            }
+        }
+
+        /* Process as many blocks as we can. */
+        while (aad_size >= BlockSize) {
+            /* Xor the current aad into our work block. */
+            for (size_t i = 0; i < BlockSize; ++i) {
+                this->block_x.block_8[BlockSize - 1 - i] ^= *(cur_aad++);
+            }
+
+            /* Multiply the blocks in our galois field. */
+            GaloisFieldMult(std::addressof(this->block_x), std::addressof(this->block_x), std::addressof(this->h_mult_blocks[0]));
+
+            /* Note that we've processed a block. */
+            aad_size -= BlockSize;
+        }
+
+        /* Update our state with whatever aad is left over. */
+        if (aad_size > 0) {
+            /* Note how much left over data we have. */
+            this->aad_remaining = static_cast<u32>(aad_size);
+
+            /* Xor the data in. */
+            for (size_t i = 0; i < aad_size; ++i) {
+                this->block_x.block_8[BlockSize - 1 - i] ^= *(cur_aad++);
+            }
+        }
+    }
+
+    /* TODO: template<class BlockCipher> size_t GcmModeImpl<BlockCipher>::UpdateEncrypt(void *dst, size_t dst_size, const void *src, size_t src_size); */
+
+    /* TODO: template<class BlockCipher> size_t GcmModeImpl<BlockCipher>::UpdateDecrypt(void *dst, size_t dst_size, const void *src, size_t src_size); */
+
+    template<class BlockCipher>
+    void GcmModeImpl<BlockCipher>::GetMac(void *dst, size_t dst_size) {
+        /* Validate pre-conditions. */
+        AMS_ASSERT(State_ProcessingAad <= this->state && this->state <= State_Done);
+        AMS_ASSERT(dst != nullptr);
+        AMS_ASSERT(dst_size >= MacSize);
+        AMS_ASSERT(this->aad_remaining == 0);
+        AMS_ASSERT(this->msg_remaining == 0);
+
+        /* If we haven't already done so, compute the final mac. */
+        if (this->state != State_Done) {
+            this->ComputeMac(true);
+            this->state = State_Done;
+        }
+
+        static_assert(sizeof(this->block_x) == MacSize);
+        std::memcpy(dst, std::addressof(this->block_x), MacSize);
+    }
+
+    template<class BlockCipher>
+    void GcmModeImpl<BlockCipher>::InitializeHashKey() {
+        /* We want to encrypt an empty block to use for intermediate calculations. */
+        /* NOTE: Non-EL1 implementations will do multiple encryptions ahead of time, */
+        /*       to speed up galois field arithmetic. */
+        constexpr const Block EmptyBlock = {};
+
+        this->ProcessBlock(std::addressof(this->h_mult_blocks[0]), std::addressof(EmptyBlock), this->block_cipher);
+    }
+
+    template<class BlockCipher>
+    void GcmModeImpl<BlockCipher>::ComputeMac(bool encrypt) {
+        /* If we have leftover data, process it. */
+        if (this->aad_remaining > 0 || this->msg_remaining > 0) {
+            GaloisFieldMult(std::addressof(this->block_x), std::addressof(this->block_x), std::addressof(this->h_mult_blocks[0]));
+        }
+
+        /* Setup the last block. */
+        Block last_block = Block{ .block_128 = { this->msg_size, this->aad_size } };
+
+        /* Multiply the last block by 8 to account for bit vs byte sizes. */
+        static_assert(offsetof(Block128, hi) == 0);
+        GaloisShiftLeft(std::addressof(last_block.block_128.hi));
+        GaloisShiftLeft(std::addressof(last_block.block_128.hi));
+        GaloisShiftLeft(std::addressof(last_block.block_128.hi));
+
+        /* Xor the data in. */
+        for (size_t i = 0; i < BlockSize; ++i) {
+            this->block_x.block_8[BlockSize - 1 - i] ^= last_block.block_8[i];
+        }
+
+        /* Perform the final multiplication. */
+        GaloisFieldMult(std::addressof(this->block_x), std::addressof(this->block_x), std::addressof(this->h_mult_blocks[0]));
+
+        /* If we need to do an encryption, do so. */
+        {
+            /* Encrypt the iv. */
+            u8 enc_result[BlockSize];
+            this->ProcessBlock(enc_result, std::addressof(this->block_ek0), this->block_cipher);
+
+            /* Xor the iv in. */
+            for (size_t i = 0; i < BlockSize; ++i) {
+                this->block_x.block_8[i] ^= enc_result[i];
+            }
+        }
+    }
+
+    /* Explicitly instantiate the valid template classes. */
+    template class GcmModeImpl<AesEncryptor128>;
+
+}
+
+#endif