Require PHP 8.2 and make core tokenization types readonly

.github/workflows/tests.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        php: ['8.1', '8.2', '8.3', '8.4']
+        php: ['8.2', '8.3', '8.4']
 
     steps:
       - name: Checkout

.php-cs-fixer.php

@@ -1,7 +1,7 @@
 <?php
 
 $finder = (new PhpCsFixer\Finder())
-    ->in(__DIR__);
+    ->in(__DIR__)->exclude('tests');
 
 return (new PhpCsFixer\Config())
     ->setParallelConfig(PhpCsFixer\Runner\Parallel\ParallelConfigFactory::detect())

README.md

@@ -18,7 +18,7 @@
 - **Zero Hard Dependencies** — Core tokenization has no required dependencies. Optional HTTP client needed only for Hub downloads.
 - **Hub Compatible** — Load tokenizers directly from Hugging Face Hub or from local files.
 - **Fully Tested** — Validated against BERT, GPT-2, Llama, Gemma, Qwen, RoBERTa, ALBERT, and more.
-- **Modern PHP** — Built for PHP 8.1+ with strict types, readonly properties, and clean interfaces.
+- **Modern PHP** — Built for PHP 8.2+ with strict types, readonly classes, and clean interfaces.
 
 ## Installation
 

composer.json

@@ -31,7 +31,7 @@
         }
     },
     "require": {
-        "php": "^8.1",
+        "php": "^8.2",
         "psr/http-client": "^1.0",
         "psr/http-factory": "^1.0",
         "php-http/discovery": "^1.19"

examples/document_chunking_pipeline.php

@@ -232,7 +232,7 @@ function createChunks(Tokenizer $tokenizer, string $text, int $maxTokens, int $o
     $originalWords = array_slice(explode(' ', $chunk), 0, 5);
     $decodedWords = explode(' ', $decoded);
 
-    $matchCount = count(array_filter($originalWords, fn ($w) => in_array(strtolower($w), array_map('strtolower', $decodedWords))));
+    $matchCount = count(array_filter($originalWords, static fn ($w) => in_array(strtolower($w), array_map('strtolower', $decodedWords))));
 
     if ($matchCount < 3) {
         echo '⚠️ Chunk '.($index + 1)." may have encoding issues\n";

examples/text_classification_preprocessing.php

@@ -138,8 +138,8 @@
     echo "  Structure: [CLS] premise [SEP] hypothesis [SEP]\n";
 
     // Type IDs distinguish between premise (0) and hypothesis (1)
-    $segment0Count = count(array_filter($encoding->typeIds, fn ($t) => 0 === $t));
-    $segment1Count = count(array_filter($encoding->typeIds, fn ($t) => 1 === $t));
+    $segment0Count = count(array_filter($encoding->typeIds, static fn ($t) => 0 === $t));
+    $segment1Count = count(array_filter($encoding->typeIds, static fn ($t) => 1 === $t));
 
     echo "  Segment A (premise) tokens: {$segment0Count}\n";
     echo "  Segment B (hypothesis) tokens: {$segment1Count}\n";

src/DataStructures/AddedToken.php

@@ -10,7 +10,7 @@
  *   - Whether they should only match single words
  *   - Whether to include any whitespace on its left or right.
  */
-class AddedToken
+readonly class AddedToken
 {
     public function __construct(
         /**

src/DataStructures/TokenLattice.php

@@ -117,6 +117,6 @@ public function tokenIds(): array
     {
         $nodes = $this->viterbi();
 
-        return array_map(fn ($x) => $x->tokenId, $nodes);
+        return array_map(static fn ($x) => $x->tokenId, $nodes);
     }
 }

src/Decoders/ByteLevelDecoder.php

@@ -302,7 +302,7 @@ protected function convertTokensToString(array $tokens): string
     {
         $text = implode('', $tokens);
         $textArray = preg_split('//u', $text, -1, \PREG_SPLIT_NO_EMPTY);
-        $byteArray = array_map(fn ($x) => self::UNICODE_TO_BYTES[$x] ?? \ord($x), $textArray);
+        $byteArray = array_map(static fn ($x) => self::UNICODE_TO_BYTES[$x] ?? \ord($x), $textArray);
         $binaryString = pack('C*', ...$byteArray);
 
         return mb_convert_encoding($binaryString, 'UTF-8');

src/Decoders/DecoderSequence.php

@@ -15,7 +15,7 @@ protected function processTokens(array $tokens): array
     {
         return array_reduce(
             $this->decoders,
-            fn (array $tokens, BaseDecoder $decoder) => $decoder->processTokens($tokens),
+            static fn (array $tokens, BaseDecoder $decoder) => $decoder->processTokens($tokens),
             $tokens
         );
     }

src/Encoding.php

@@ -7,16 +7,16 @@
 /**
  * Represents the output of tokenization.
  */
-class Encoding
+readonly class Encoding
 {
     /**
      * @param int[]    $ids     The list of token IDs
      * @param string[] $tokens  The list of tokens
      * @param int[]    $typeIds The list of type IDs
      */
     public function __construct(
-        public readonly array $ids,
-        public readonly array $tokens,
-        public readonly array $typeIds = [],
+        public array $ids,
+        public array $tokens,
+        public array $typeIds = [],
     ) {}
 }

src/Factories/NormalizerFactory.php

@@ -35,7 +35,7 @@ public static function create(array $config): NormalizerInterface
             ),
             'Lowercase' => new LowercaseNormalizer(),
             'Sequence' => new NormalizerSequence(
-                array_map(fn ($c) => self::create($c), $config['normalizers'] ?? [])
+                array_map(static fn ($c) => self::create($c), $config['normalizers'] ?? [])
             ),
             'Strip' => new StripNormalizer(
                 stripLeft: $config['strip_left'] ?? true,

src/Factories/PostProcessorFactory.php

@@ -45,7 +45,7 @@ public static function create(array $config): ?PostProcessorInterface
                 pair: $config['pair']
             ),
             'Sequence' => new PostProcessorSequence(
-                array_map(fn ($c) => self::create($c), $config['processors'] ?? [])
+                array_map(static fn ($c) => self::create($c), $config['processors'] ?? [])
             ),
             default => throw new \Exception("Unknown post-processor type: {$type}"),
         };

src/Factories/PreTokenizerFactory.php

@@ -55,7 +55,7 @@ public static function create(array $config): PreTokenizerInterface
                 invert: $config['invert'] ?? true
             ),
             'Sequence' => new PreTokenizerSequence(
-                array_map(fn ($c) => self::create($c), $config['pretokenizers'] ?? [])
+                array_map(static fn ($c) => self::create($c), $config['pretokenizers'] ?? [])
             ),
             default => throw new \Exception("Unknown pre-tokenizer type: {$type}"),
         };

src/Models/AbstractModel.php

@@ -45,7 +45,7 @@ public function decode(array $ids): array
             $tokens[] = $this->vocab[$id] ?? $this->unkToken ?? null;
         }
 
-        return array_filter($tokens, fn ($t) => null !== $t);
+        return array_filter($tokens, static fn ($t) => null !== $t);
     }
 
     /**
@@ -65,7 +65,7 @@ public function encode(array $tokens): array
         }
 
         // Removing nulls in case unkTokenId is null (though should exist)
-        return array_filter($ids, fn ($id) => null !== $id);
+        return array_filter($ids, static fn ($id) => null !== $id);
     }
 
     /**

src/Models/BPEModel.php

@@ -59,7 +59,7 @@ public function __construct(
             $this->merges = $merges;
         } else {
             $this->merges = array_map(
-                fn ($merge) => \is_string($merge) ? explode(' ', $merge, 2) : $merge,
+                static fn ($merge) => \is_string($merge) ? explode(' ', $merge, 2) : $merge,
                 $merges
             );
         }

src/Normalizers/NormalizerSequence.php

@@ -17,7 +17,7 @@ public function normalize(string $text): string
     {
         return array_reduce(
             $this->normalizers,
-            fn (string $text, NormalizerInterface $normalizer) => $normalizer->normalize($text),
+            static fn (string $text, NormalizerInterface $normalizer) => $normalizer->normalize($text),
             $text
         );
     }

src/PreTokenizers/ByteLevelPreTokenizer.php

@@ -303,9 +303,9 @@ public function preTokenize(array|string $text, array $options = []): array
         }
 
         // Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
-        return array_map(function ($token) {
+        return array_map(static function ($token) {
             $utf8Bytes = mb_convert_encoding($token, 'UTF-8');
-            $bytes = array_map(fn ($byte) => self::BYTES_TO_UNICODE[$byte], unpack('C*', $utf8Bytes));
+            $bytes = array_map(static fn ($byte) => self::BYTES_TO_UNICODE[$byte], unpack('C*', $utf8Bytes));
 
             return implode('', $bytes);
         }, $tokens);

src/PreTokenizers/PreTokenizerSequence.php

@@ -17,7 +17,7 @@ public function preTokenize(array|string $text, array $options = []): array
     {
         return array_reduce(
             $this->preTokenizers,
-            fn ($text, PreTokenizerInterface $preTokenizer) => $preTokenizer->preTokenize($text, $options),
+            static fn ($text, PreTokenizerInterface $preTokenizer) => $preTokenizer->preTokenize($text, $options),
             \is_array($text) ? $text : [$text]
         );
     }

src/Tokenizer.php

@@ -25,12 +25,12 @@
 use Codewithkyrian\Tokenizers\Utils\DecoderUtils;
 use Codewithkyrian\Tokenizers\Utils\NormalizerUtils;
 
-class Tokenizer
+readonly class Tokenizer
 {
     /**
      * The model's maximum sequence length (convenience accessor for config).
      */
-    public readonly ?int $modelMaxLength;
+    public ?int $modelMaxLength;
     protected DictionarySplitter $addedTokensSplitter;
 
     /**
@@ -44,11 +44,11 @@ class Tokenizer
      * @param array<string, mixed>      $config        Additional configuration options
      */
     public function __construct(
-        protected ModelInterface $model,
-        protected NormalizerInterface $normalizer,
-        protected PreTokenizerInterface $preTokenizer,
-        protected PostProcessorInterface $postProcessor,
-        protected DecoderInterface $decoder,
+        public ModelInterface $model,
+        public NormalizerInterface $normalizer,
+        public PreTokenizerInterface $preTokenizer,
+        public PostProcessorInterface $postProcessor,
+        public DecoderInterface $decoder,
         protected array $specialTokens = [],
         protected array $addedTokens = [],
         protected array $config = []

tests/Pest.php

@@ -9,7 +9,7 @@
  */
 function modelTokenizationDataset(string $datasetClass, bool $withTextPair = false): Closure
 {
-    return function () use ($datasetClass, $withTextPair) {
+    return static function () use ($datasetClass, $withTextPair) {
         if (!class_exists($datasetClass) || !method_exists($datasetClass, 'data')) {
             return;
         }