Implement configuration access and reconstruction for Tokenizer

src/Contracts/DecoderInterface.php

@@ -12,4 +12,14 @@ interface DecoderInterface
      * @param string[] $tokens
      */
     public function decode(array $tokens): string;
+
+    /**
+     * Get configuration value(s).
+     *
+     * @param null|string $key     The configuration key. If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist
+     *
+     * @return mixed the configuration value, or full config array if $key is null
+     */
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/Contracts/ModelInterface.php

@@ -54,10 +54,12 @@ public function getVocabSize(): int;
     public function addToken(string $token, int $id): void;
 
     /**
-     * Get the end of word suffix, if any.
-     * Only some models (like BPE) have this property.
+     * Get configuration value(s).
      *
-     * @return null|string the end of word suffix
+     * @param null|string $key     The configuration key (e.g., 'dropout'). If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist (ignored when $key is null)
+     *
+     * @return mixed the configuration value, or full config array if $key is null
      */
-    public function getEndOfWordSuffix(): ?string;
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/Contracts/NormalizerInterface.php

@@ -7,4 +7,14 @@
 interface NormalizerInterface
 {
     public function normalize(string $text): string;
+
+    /**
+     * Get configuration value(s).
+     *
+     * @param null|string $key     The configuration key. If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist
+     *
+     * @return mixed the configuration value, or full config array if $key is null
+     */
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/Contracts/PostProcessorInterface.php

@@ -15,4 +15,14 @@ interface PostProcessorInterface
      * @return array{0: string[], 1: int[]} the processed tokens and type IDs
      */
     public function process(array $tokens, ?array $pair = null, bool $addSpecialTokens = true): array;
+
+    /**
+     * Get configuration value(s).
+     *
+     * @param null|string $key     The configuration key. If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist
+     *
+     * @return mixed the configuration value, or full config array if $key is null
+     */
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/Contracts/PreTokenizerInterface.php

@@ -15,4 +15,14 @@ interface PreTokenizerInterface
      * @return string[]
      */
     public function preTokenize(array|string $text, array $options = []): array;
+
+    /**
+     * Get configuration value(s).
+     *
+     * @param null|string $key     The configuration key. If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist
+     *
+     * @return mixed the configuration value, or full config array if $key is null
+     */
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/DataStructures/AddedToken.php

@@ -10,7 +10,7 @@
  *   - Whether they should only match single words
  *   - Whether to include any whitespace on its left or right.
  */
-readonly class AddedToken
+class AddedToken implements \JsonSerializable
 {
     public function __construct(
         /**
@@ -58,4 +58,20 @@ public static function fromArray(array $data): self
             $data['special'] ?? false,
         );
     }
+
+    /**
+     * @return array<string, mixed>
+     */
+    public function jsonSerialize(): array
+    {
+        return [
+            'id' => $this->id,
+            'content' => $this->content,
+            'single_word' => $this->singleWord,
+            'lstrip' => $this->lStrip,
+            'rstrip' => $this->rStrip,
+            'normalized' => $this->normalized,
+            'special' => $this->special,
+        ];
+    }
 }

src/Decoders/BPEDecoder.php

@@ -8,6 +8,22 @@ class BPEDecoder extends BaseDecoder
 {
     public function __construct(protected string $suffix = '') {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'BPEDecoder',
+                'suffix' => $this->suffix,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'BPEDecoder',
+            'suffix' => $this->suffix,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         $decoded = [];

src/Decoders/ByteFallbackDecoder.php

@@ -6,6 +6,15 @@
 
 class ByteFallbackDecoder extends BaseDecoder
 {
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return 'type' === $key ? 'ByteFallback' : $default;
+        }
+
+        return ['type' => 'ByteFallback'];
+    }
+
     protected function processTokens(array $tokens): array
     {
         $newTokens = [];

src/Decoders/ByteLevelDecoder.php

@@ -275,11 +275,16 @@ public function __construct(protected array $addedTokens = [], protected ?string
 
     /**
      * Convert an array of tokens to a string by decoding each byte.
-     *
-     * @param string[] $tokens array of tokens to be decoded
-     *
-     * @return string the decoded string
      */
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return 'type' === $key ? 'ByteLevel' : $default;
+        }
+
+        return ['type' => 'ByteLevel'];
+    }
+
     public function decode(array $tokens): string
     {
         $decoded = parent::decode($tokens);

src/Decoders/CTCDecoder.php

@@ -14,6 +14,26 @@ public function __construct(
         protected bool $cleanup = true
     ) {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'CTC',
+                'pad_token' => $this->padToken,
+                'word_delimiter_token' => $this->wordDelimiterToken,
+                'cleanup' => $this->cleanup,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'CTC',
+            'pad_token' => $this->padToken,
+            'word_delimiter_token' => $this->wordDelimiterToken,
+            'cleanup' => $this->cleanup,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         if (empty($tokens)) {

src/Decoders/DecoderSequence.php

@@ -11,6 +11,22 @@ class DecoderSequence extends BaseDecoder
      */
     public function __construct(protected array $decoders) {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'Sequence',
+                'decoders' => array_map(static fn (BaseDecoder $d) => $d->getConfig(), $this->decoders),
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'Sequence',
+            'decoders' => array_map(static fn (BaseDecoder $d) => $d->getConfig(), $this->decoders),
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         return array_reduce(

src/Decoders/FuseDecoder.php

@@ -14,6 +14,22 @@ public function __construct(
         protected string $separator = ''
     ) {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'Fuse',
+                'separator' => $this->separator,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'Fuse',
+            'separator' => $this->separator,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         return [implode($this->separator, $tokens)];

src/Decoders/MetaspaceDecoder.php

@@ -11,6 +11,24 @@ public function __construct(
         protected bool $addPrefixSpace = true
     ) {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'Metaspace',
+                'replacement' => $this->replacement,
+                'add_prefix_space' => $this->addPrefixSpace,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'Metaspace',
+            'replacement' => $this->replacement,
+            'add_prefix_space' => $this->addPrefixSpace,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         $result = [];

src/Decoders/ReplaceDecoder.php

@@ -12,6 +12,24 @@ public function __construct(
         protected string $replacement = ''
     ) {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'Replace',
+                'pattern' => $this->regex ? ['Regex' => $this->regex] : ['String' => $this->subString],
+                'content' => $this->replacement,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'Replace',
+            'pattern' => $this->regex ? ['Regex' => $this->regex] : ['String' => $this->subString],
+            'content' => $this->replacement,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         return array_map(function ($token) {

src/Decoders/StripDecoder.php

@@ -12,6 +12,26 @@ public function __construct(
         protected int $stop
     ) {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'Strip',
+                'content' => $this->content,
+                'start' => $this->start,
+                'stop' => $this->stop,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'Strip',
+            'content' => $this->content,
+            'start' => $this->start,
+            'stop' => $this->stop,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         return array_map(function ($token) {

src/Decoders/WordPieceDecoder.php

@@ -13,6 +13,24 @@ public function __construct(
         protected bool $cleanup = true
     ) {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'WordPiece',
+                'prefix' => $this->prefix,
+                'cleanup' => $this->cleanup,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'WordPiece',
+            'prefix' => $this->prefix,
+            'cleanup' => $this->cleanup,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         $decodedTokens = [];

src/Factories/DecoderFactory.php

@@ -25,10 +25,10 @@ class DecoderFactory
      * @param array<string, AddedToken> $addedTokens     Optional. Only needed for ByteLevelDecoder.
      * @param null|string               $endOfWordSuffix Optional. Only needed for ByteLevelDecoder.
      */
-    public static function create(array $config, array $addedTokens = [], ?string $endOfWordSuffix = null): ?DecoderInterface
+    public static function create(array $config, array $addedTokens = [], ?string $endOfWordSuffix = null): DecoderInterface
     {
         if (empty($config)) {
-            return null;
+            return new FuseDecoder(' ');
         }
 
         $type = $config['type'] ?? null;

src/Factories/NormalizerFactory.php

@@ -11,6 +11,7 @@
 use Codewithkyrian\Tokenizers\Normalizers\NFKCNormalizer;
 use Codewithkyrian\Tokenizers\Normalizers\NFKDNormalizer;
 use Codewithkyrian\Tokenizers\Normalizers\NormalizerSequence;
+use Codewithkyrian\Tokenizers\Normalizers\PassThroughNormalizer;
 use Codewithkyrian\Tokenizers\Normalizers\PrecompiledNormalizer;
 use Codewithkyrian\Tokenizers\Normalizers\PrependNormalizer;
 use Codewithkyrian\Tokenizers\Normalizers\ReplaceNormalizer;
@@ -24,6 +25,10 @@ class NormalizerFactory
      */
     public static function create(array $config): NormalizerInterface
     {
+        if (empty($config)) {
+            return new PassThroughNormalizer();
+        }
+
         $type = $config['type'] ?? null;
 
         return match ($type) {

src/Factories/PostProcessorFactory.php

@@ -7,6 +7,7 @@
 use Codewithkyrian\Tokenizers\Contracts\PostProcessorInterface;
 use Codewithkyrian\Tokenizers\PostProcessors\BertPostProcessor;
 use Codewithkyrian\Tokenizers\PostProcessors\ByteLevelPostProcessor;
+use Codewithkyrian\Tokenizers\PostProcessors\DefaultPostProcessor;
 use Codewithkyrian\Tokenizers\PostProcessors\PostProcessorSequence;
 use Codewithkyrian\Tokenizers\PostProcessors\RobertaPostProcessor;
 use Codewithkyrian\Tokenizers\PostProcessors\TemplatePostProcessor;
@@ -16,10 +17,10 @@ class PostProcessorFactory
     /**
      * @param array<string, mixed> $config the post-processor configuration
      */
-    public static function create(array $config): ?PostProcessorInterface
+    public static function create(array $config): PostProcessorInterface
     {
         if (empty($config)) {
-            return null;
+            return new DefaultPostProcessor();
         }
 
         $type = $config['type'] ?? null;