From e7f55a4cbd8b55cb98fcc99e09b288e4466623a4 Mon Sep 17 00:00:00 2001 From: Kyrian Obikwelu Date: Mon, 22 Apr 2024 17:37:45 +0100 Subject: [PATCH 1/4] Use a custom Inference session class to make Tensor a first class output --- composer.json | 3 +- examples/composer.json | 59 +- examples/pipelines/text-generation.php | 4 +- src/Models/ModelArchitecture.php | 3 - .../BartForConditionalGeneration.php | 2 +- src/Models/Pretrained/GPT2PretrainedModel.php | 2 +- .../M2M100ForConditionalGeneration.php | 2 +- src/Models/Pretrained/PretrainedModel.php | 22 +- .../Pretrained/Qwen2PreTrainedModel.php | 4 +- .../Pretrained/T5ForConditionalGeneration.php | 2 +- .../Pretrained/VisionEncoderDecoderModel.php | 2 +- src/Pipelines/TextGenerationPipeline.php | 2 +- src/Utils/Helpers.php | 5 +- src/Utils/InferenceSession.php | 724 ++++++++++++++++++ src/Utils/Tensor.php | 86 ++- 15 files changed, 841 insertions(+), 81 deletions(-) create mode 100644 src/Utils/InferenceSession.php diff --git a/composer.json b/composer.json index b53eea9..4a56ef0 100644 --- a/composer.json +++ b/composer.json @@ -21,7 +21,8 @@ "codewithkyrian/onnxruntime-downloader-plugin": "^1.1", "symfony/console": "^6.4|^7.0", "imagine/imagine": "^1.3", - "rokka/imagine-vips": "^0.31.0" + "rokka/imagine-vips": "^0.31.0", + "spatie/fork": "^1.2" }, "require-dev": { "pestphp/pest": "^2.31", diff --git a/examples/composer.json b/examples/composer.json index 12e5e84..7195f76 100644 --- a/examples/composer.json +++ b/examples/composer.json @@ -1,29 +1,34 @@ { - "name": "kyrian/examples", - "autoload": { - "psr-4": { - "Kyrian\\Examples\\": "/" - } - }, - "authors": [ - { - "name": "Kyrian Obikwelu", - "email": "koshnawaza@gmail.com" - } - ], - "require": { - "php": "^8.1", - "symfony/console": "^7.0", - "codewithkyrian/transformers": "dev-change-init-process" - }, - "minimum-stability": "dev", - "require-dev": { - "symfony/var-dumper": "^7.0" - }, - "repositories": [ - { - "type" : "path", - "url": "../" - } - ] + "name": "kyrian/examples", + "autoload": { + "psr-4": { + "Kyrian\\Examples\\": "/" + } + }, + "authors": [ + { + "name": "Kyrian Obikwelu", + "email": "koshnawaza@gmail.com" + } + ], + "require": { + "php": "^8.1", + "symfony/console": "^7.0", + "codewithkyrian/transformers": "dev-main" + }, + "minimum-stability": "dev", + "require-dev": { + "symfony/var-dumper": "^7.0" + }, + "repositories": [ + { + "type": "path", + "url": "../" + } + ], + "config": { + "allow-plugins": { + "codewithkyrian/onnxruntime-downloader-plugin": true + } + } } diff --git a/examples/pipelines/text-generation.php b/examples/pipelines/text-generation.php index 841694e..e8e4ee9 100644 --- a/examples/pipelines/text-generation.php +++ b/examples/pipelines/text-generation.php @@ -18,13 +18,13 @@ $messages = [ ['role' => 'system', 'content' => 'You are a helpful assistant.'], - ['role' => 'user', 'content' => 'What is the product of 5 and 4'], + ['role' => 'user', 'content' => 'What is diffusion?'], ]; $input = $generator->tokenizer->applyChatTemplate($messages, addGenerationPrompt: true, tokenize: false); $output = $generator($input, - streamer: $streamer, +// streamer: $streamer, maxNewTokens: 128, doSample: true, returnFullText: false, diff --git a/src/Models/ModelArchitecture.php b/src/Models/ModelArchitecture.php index b1c71b0..b502330 100644 --- a/src/Models/ModelArchitecture.php +++ b/src/Models/ModelArchitecture.php @@ -119,8 +119,6 @@ protected function decoderRunBeam(PretrainedModel $model, array &$beam): array 'past_key_values' => $beam['prev_model_outputs']['past_key_values'] ?? null, ]; - - // 2. Run $output = $model->forward($modelInputs); // 3. Update @@ -372,7 +370,6 @@ protected function seq2seqForward(PretrainedModel $model, array $modelInputs): a $model->addPastKeyValues($decoderFeeds, $pastKeyValues); $decoderResults = $model->runSession($model->decoderMergedSession, $decoderFeeds); - $logits = $decoderResults['logits']; $pastKeyValues = $model->getPastKeyValues($decoderResults, $pastKeyValues); diff --git a/src/Models/Pretrained/BartForConditionalGeneration.php b/src/Models/Pretrained/BartForConditionalGeneration.php index ad04b28..bae788f 100644 --- a/src/Models/Pretrained/BartForConditionalGeneration.php +++ b/src/Models/Pretrained/BartForConditionalGeneration.php @@ -8,7 +8,7 @@ use Codewithkyrian\Transformers\Models\ModelArchitecture; use Codewithkyrian\Transformers\Utils\AutoConfig; use Codewithkyrian\Transformers\Utils\GenerationConfig; -use OnnxRuntime\InferenceSession; +use Codewithkyrian\Transformers\Utils\InferenceSession; /** * The BART Model with a language modeling head. Can be used for summarization. diff --git a/src/Models/Pretrained/GPT2PretrainedModel.php b/src/Models/Pretrained/GPT2PretrainedModel.php index e3f6ebf..277631f 100644 --- a/src/Models/Pretrained/GPT2PretrainedModel.php +++ b/src/Models/Pretrained/GPT2PretrainedModel.php @@ -8,7 +8,7 @@ use Codewithkyrian\Transformers\Models\ModelArchitecture; use Codewithkyrian\Transformers\Utils\AutoConfig; use Codewithkyrian\Transformers\Utils\GenerationConfig; -use OnnxRuntime\InferenceSession; +use Codewithkyrian\Transformers\Utils\InferenceSession; class GPT2PretrainedModel extends PretrainedModel { diff --git a/src/Models/Pretrained/M2M100ForConditionalGeneration.php b/src/Models/Pretrained/M2M100ForConditionalGeneration.php index cc88e51..1796b08 100644 --- a/src/Models/Pretrained/M2M100ForConditionalGeneration.php +++ b/src/Models/Pretrained/M2M100ForConditionalGeneration.php @@ -8,7 +8,7 @@ use Codewithkyrian\Transformers\Models\ModelArchitecture; use Codewithkyrian\Transformers\Utils\AutoConfig; use Codewithkyrian\Transformers\Utils\GenerationConfig; -use OnnxRuntime\InferenceSession; +use Codewithkyrian\Transformers\Utils\InferenceSession; class M2M100ForConditionalGeneration extends M2M100PretrainedModel { diff --git a/src/Models/Pretrained/PretrainedModel.php b/src/Models/Pretrained/PretrainedModel.php index 1859bd5..eb2b93a 100644 --- a/src/Models/Pretrained/PretrainedModel.php +++ b/src/Models/Pretrained/PretrainedModel.php @@ -27,12 +27,13 @@ use Codewithkyrian\Transformers\Utils\AutoConfig; use Codewithkyrian\Transformers\Utils\GenerationConfig; use Codewithkyrian\Transformers\Utils\Hub; +use Codewithkyrian\Transformers\Utils\InferenceSession; use Codewithkyrian\Transformers\Utils\Tensor; use Error; use Exception; -use OnnxRuntime\InferenceSession; use Symfony\Component\Console\Output\OutputInterface; use function Codewithkyrian\Transformers\Utils\array_some; +use function Codewithkyrian\Transformers\Utils\timeUsage; /** * A base class for pre-trained models that provides the model configuration and an ONNX session. @@ -281,9 +282,10 @@ public function runSession(InferenceSession $session, array $inputs): array $outputNames = array_column($session->outputs(), 'name'); - $outputs = $session->run($outputNames, $inputs); - - return array_combine($outputNames, array_map([Tensor::class, 'fromArray'], $outputs)); + timeUsage(); + $out = $session->run($outputNames, $inputs); + dump(timeUsage(true)); + return $out; } catch (MissingModelInputException $e) { throw $e; } catch (Exception $e) { @@ -331,7 +333,8 @@ public function validateInputs(array $inputNames, array $inputs): array The following inputs will be ignored: "' . implode(', ', $ignored) . '".'; } - return array_map(fn($i) => $i->toArray(), $inputs); +// return array_map(fn($i) => $i->toArray(), $inputs); + return $inputs; } /** @@ -521,8 +524,10 @@ public function addPastKeyValues(array &$decoderFeeds, ?array $pastKeyValues): v * @param Tensor $inputs The input token ids. * @param GenerationConfig|null $generationConfig The generation configuration to use. If null, default configuration will be used. * @param LogitsProcessorList|null $logitsProcessor An optional logits processor to use. If null, a new LogitsProcessorList instance will be created. - * @param array|null $inputsAttentionMask An optional attention mask for the inputs. + * @param Tensor|null $inputsAttentionMask An optional attention mask for the inputs. + * @param Streamer|null $streamer * @return array An array of generated output sequences, where each sequence is an array of token IDs. + * @throws Exception */ public function generate( Tensor $inputs, @@ -609,6 +614,7 @@ public function generate( $output = $this->runBeam($beam); + // add attentions/scores to beam only if user requested if ($generationConfig->output_attentions) { $this->addAttentionsToBeam($beam, $output); @@ -626,6 +632,7 @@ public function generate( $logits = $output['logits']->slice(null, -1, null); // $logits = $output['logits']; + // Apply logits processor $logitsProcessor($beam['output_token_ids'], $logits); @@ -649,7 +656,6 @@ public function generate( } - ++$numOutputTokens; // Group and select best beams @@ -665,7 +671,6 @@ function ($group) use ($generationConfig) { $this->groupBeams($newestBeams) )); - // Flatten beams $beams = $newestBeams; @@ -673,7 +678,6 @@ function ($group) use ($generationConfig) { $streamer?->put($beams); } - // TODO: Ensure that we can return non-batched outputs $groupedBeams = $this->groupBeams($beams); diff --git a/src/Models/Pretrained/Qwen2PreTrainedModel.php b/src/Models/Pretrained/Qwen2PreTrainedModel.php index 3418f08..bcefda2 100644 --- a/src/Models/Pretrained/Qwen2PreTrainedModel.php +++ b/src/Models/Pretrained/Qwen2PreTrainedModel.php @@ -8,7 +8,7 @@ use Codewithkyrian\Transformers\Models\ModelArchitecture; use Codewithkyrian\Transformers\Utils\AutoConfig; use Codewithkyrian\Transformers\Utils\GenerationConfig; -use OnnxRuntime\InferenceSession; +use Codewithkyrian\Transformers\Utils\InferenceSession; /** * The bare Qwen2 Model outputting raw hidden-states without any specific head on top. @@ -32,7 +32,7 @@ public function __construct( $this->config['pad_token_id'] = $this->config['eos_token_id']; $this->config->padTokenId = $this->config['eos_token_id']; - $this->numHeads = $this->config['num_key_value_heads'] ?? $this->config['num_attention_heads']; + $this->numHeads = $this->config['num_key_value_heads'] ?? $this->config['num_attention_heads']; $this->numLayers = $this->config['num_hidden_layers']; $this->dimKv = $this->config['hidden_size'] / $this->config['num_attention_heads']; } diff --git a/src/Models/Pretrained/T5ForConditionalGeneration.php b/src/Models/Pretrained/T5ForConditionalGeneration.php index 9b884c4..8087961 100644 --- a/src/Models/Pretrained/T5ForConditionalGeneration.php +++ b/src/Models/Pretrained/T5ForConditionalGeneration.php @@ -8,7 +8,7 @@ use Codewithkyrian\Transformers\Models\ModelArchitecture; use Codewithkyrian\Transformers\Utils\AutoConfig; use Codewithkyrian\Transformers\Utils\GenerationConfig; -use OnnxRuntime\InferenceSession; +use Codewithkyrian\Transformers\Utils\InferenceSession; /** * T5Model is a class representing a T5 model for conditional generation. diff --git a/src/Models/Pretrained/VisionEncoderDecoderModel.php b/src/Models/Pretrained/VisionEncoderDecoderModel.php index c55a6b7..e0f2900 100644 --- a/src/Models/Pretrained/VisionEncoderDecoderModel.php +++ b/src/Models/Pretrained/VisionEncoderDecoderModel.php @@ -11,7 +11,7 @@ use Codewithkyrian\Transformers\Models\ModelArchitecture; use Codewithkyrian\Transformers\Utils\AutoConfig; use Codewithkyrian\Transformers\Utils\GenerationConfig; -use OnnxRuntime\InferenceSession; +use Codewithkyrian\Transformers\Utils\InferenceSession; /** * Vision Encoder-Decoder model based on OpenAI's GPT architecture for image captioning and other vision tasks diff --git a/src/Pipelines/TextGenerationPipeline.php b/src/Pipelines/TextGenerationPipeline.php index 716736d..c464379 100644 --- a/src/Pipelines/TextGenerationPipeline.php +++ b/src/Pipelines/TextGenerationPipeline.php @@ -114,7 +114,7 @@ public function __invoke(array|string $inputs, ...$args): array ); // Streamer can only handle one input at a time for now, so we only pass the first input - $streamer->init($this->tokenizer, $inputIds[0]->toArray(), true); + $streamer?->init($this->tokenizer, $inputIds[0]->toArray(), true); $outputTokenIds = $this->model->generate($inputIds, generationConfig: $generationConfig, diff --git a/src/Utils/Helpers.php b/src/Utils/Helpers.php index 7c3672b..f508481 100644 --- a/src/Utils/Helpers.php +++ b/src/Utils/Helpers.php @@ -21,7 +21,7 @@ function memoryPeak(): string } -function timeUsage(bool $milliseconds = false, bool $sinceLastCall = true): string +function timeUsage(bool $milliseconds = false, bool $sinceLastCall = true, bool $returnString = true): string|float { static $lastCallTime = 0; @@ -35,7 +35,8 @@ function timeUsage(bool $milliseconds = false, bool $sinceLastCall = true): stri $timeDiff = $milliseconds ? $timeDiff * 1000 : $timeDiff; - return @round($timeDiff, 4) . ($milliseconds ? ' ms' : ' s'); +// return @round($timeDiff, 4) . ($milliseconds ? ' ms' : ' s'); + return $returnString ? @round($timeDiff, 4) . ($milliseconds ? ' ms' : ' s') : @round($timeDiff, 4); } function array_some(array $array, callable $callback): bool diff --git a/src/Utils/InferenceSession.php b/src/Utils/InferenceSession.php new file mode 100644 index 0000000..911053d --- /dev/null +++ b/src/Utils/InferenceSession.php @@ -0,0 +1,724 @@ +ffi = FFI::instance(); + $this->api = self::api(); + + // session options + $sessionOptions = $this->ffi->new('OrtSessionOptions*'); + $this->checkStatus(($this->api->CreateSessionOptions)(\FFI::addr($sessionOptions))); + if ($enableCpuMemArena) { + $this->checkStatus(($this->api->EnableCpuMemArena)($sessionOptions)); + } else { + $this->checkStatus(($this->api->DisableCpuMemArena)($sessionOptions)); + } + if ($enableMemPattern) { + $this->checkStatus(($this->api->EnableMemPattern)($sessionOptions)); + } else { + $this->checkStatus(($this->api->DisableMemPattern)($sessionOptions)); + } + if ($enableProfiling) { + $this->checkStatus(($this->api->EnableProfiling)($sessionOptions, $this->ortString($profileFilePrefix ?? 'onnxruntime_profile_'))); + } else { + $this->checkStatus(($this->api->DisableProfiling)($sessionOptions)); + } + if (!is_null($executionMode)) { + $this->checkStatus(($this->api->SetSessionExecutionMode)($sessionOptions, $executionMode->value)); + } + if (!is_null($freeDimensionOverridesByDenotation)) { + foreach ($freeDimensionOverridesByDenotation as $k => $v) { + $this->checkStatus(($this->api->AddFreeDimensionOverride)($sessionOptions, $k, $v)); + } + } + if (!is_null($freeDimensionOverridesByName)) { + foreach ($freeDimensionOverridesByName as $k => $v) { + $this->checkStatus(($this->api->AddFreeDimensionOverrideByName)($sessionOptions, $k, $v)); + } + } + if (!is_null($graphOptimizationLevel)) { + $this->checkStatus(($this->api->SetSessionGraphOptimizationLevel)($sessionOptions, $graphOptimizationLevel->value)); + } + if (!is_null($interOpNumThreads)) { + $this->checkStatus(($this->api->SetInterOpNumThreads)($sessionOptions, $interOpNumThreads)); + } + if (!is_null($intraOpNumThreads)) { + $this->checkStatus(($this->api->SetIntraOpNumThreads)($sessionOptions, $intraOpNumThreads)); + } + if (!is_null($logSeverityLevel)) { + $this->checkStatus(($this->api->SetSessionLogSeverityLevel)($sessionOptions, $logSeverityLevel)); + } + if (!is_null($logVerbosityLevel)) { + $this->checkStatus(($this->api->SetSessionLogVerbosityLevel)($sessionOptions, $logVerbosityLevel)); + } + if (!is_null($logid)) { + $this->checkStatus(($this->api->SetSessionLogId)($sessionOptions, $logid)); + } + if (!is_null($optimizedModelFilepath)) { + $this->checkStatus(($this->api->SetOptimizedModelFilePath)($sessionOptions, $this->ortString($optimizedModelFilepath))); + } + if (!is_null($sessionConfigEntries)) { + foreach ($sessionConfigEntries as $k => $v) { + $this->checkStatus(($this->api->AddSessionConfigEntry)($sessionOptions, $k, $v)); + } + } + foreach ($providers as $provider) { + if (!in_array($provider, $this->providers())) { + trigger_error('Provider not available: ' . $provider, E_USER_WARNING); + continue; + } + + if ($provider == 'CUDAExecutionProvider') { + $cudaOptions = $this->ffi->new('OrtCUDAProviderOptionsV2*'); + $this->checkStatus(($this->api->CreateCUDAProviderOptions)(\FFI::addr($cudaOptions))); + $this->checkStatus(($this->api->SessionOptionsAppendExecutionProvider_CUDA_V2)($sessionOptions, $cudaOptions)); + ($this->api->ReleaseCUDAProviderOptions)($cudaOptions); + } elseif ($provider == 'CPUExecutionProvider') { + break; + } else { + throw new \InvalidArgumentException('Provider not supported: ' . $provider); + } + } + + $this->session = $this->loadSession($path, $sessionOptions); + $this->allocator = $this->loadAllocator(); + $this->inputs = $this->loadInputs(); + $this->outputs = $this->loadOutputs(); + + ($this->api->ReleaseSessionOptions)($sessionOptions); + } + + public function __destruct() + { + ($this->api->ReleaseSession)($this->session); + } + + public function run($outputNames, $inputFeed, $logSeverityLevel = null, $logVerbosityLevel = null, $logid = null, $terminate = null): array + { + // pointer references + $refs = []; + + $inputTensor = $this->createInputTensor($inputFeed, $refs); + + $outputNames ??= array_map(fn($v) => $v['name'], $this->outputs); + + $outputsSize = count($outputNames); + $outputTensor = $this->ffi->new("OrtValue*[$outputsSize]"); + $inputNodeNames = $this->createNodeNames(array_keys($inputFeed), $refs); + $outputNodeNames = $this->createNodeNames($outputNames, $refs); + + // run options + $runOptions = $this->ffi->new('OrtRunOptions*'); + $this->checkStatus(($this->api->CreateRunOptions)(\FFI::addr($runOptions))); + if (!is_null($logVerbosityLevel)) { + $this->checkStatus(($this->api->RunOptionsSetRunLogSeverityLevel)($runOptions, $logSeverityLevel)); + } + if (!is_null($logVerbosityLevel)) { + $this->checkStatus(($this->api->RunOptionsSetRunLogVerbosityLevel)($runOptions, $logVerbosityLevel)); + } + if (!is_null($logid)) { + $this->checkStatus(($this->api->RunOptionsSetRunTag)($runOptions, $logid)); + } + if (!is_null($terminate)) { + if ($terminate) { + $this->checkStatus(($this->api->RunOptionsSetTerminate)($runOptions)); + } else { + $this->checkStatus(($this->api->RunOptionsUnsetTerminate)($runOptions)); + } + } + + $this->checkStatus(($this->api->Run)($this->session, $runOptions, $inputNodeNames, $inputTensor, count($inputFeed), $outputNodeNames, count($outputNames), $outputTensor)); + + $output = []; + + foreach ($outputTensor as $i => $t) { +// $output[] = $this->createFromOnnxValue($t); + $output[$outputNames[$i]] = $this->createFromOnnxValue($t); + } + + // TODO use finally + ($this->api->ReleaseRunOptions)($runOptions); + if ($inputTensor) { + for ($i = 0; $i < count($inputFeed); $i++) { + ($this->api->ReleaseValue)($inputTensor[$i]); + } + } + // output values released in createFromOnnxValue + + return $output; + } + + public function inputs() + { + return $this->inputs; + } + + public function outputs() + { + return $this->outputs; + } + + public function modelmeta() + { + $keys = $this->ffi->new('char**'); + $numKeys = $this->ffi->new('int64_t'); + $description = $this->ffi->new('char*'); + $domain = $this->ffi->new('char*'); + $graphName = $this->ffi->new('char*'); + $graphDescription = $this->ffi->new('char*'); + $producerName = $this->ffi->new('char*'); + $version = $this->ffi->new('int64_t'); + + $metadata = $this->ffi->new('OrtModelMetadata*'); + $this->checkStatus(($this->api->SessionGetModelMetadata)($this->session, \FFI::addr($metadata))); + + $customMetadataMap = []; + $this->checkStatus(($this->api->ModelMetadataGetCustomMetadataMapKeys)($metadata, $this->allocator, \FFI::addr($keys), \FFI::addr($numKeys))); + for ($i = 0; $i < $numKeys->cdata; $i++) { + $keyPtr = $keys[$i]; + $key = \FFI::string($keyPtr); + $value = $this->ffi->new('char*'); + $this->checkStatus(($this->api->ModelMetadataLookupCustomMetadataMap)($metadata, $this->allocator, $key, \FFI::addr($value))); + $customMetadataMap[$key] = \FFI::string($value); + + $this->allocatorFree($keyPtr); + $this->allocatorFree($value); + } + $this->allocatorFree($keys); + + $this->checkStatus(($this->api->ModelMetadataGetDescription)($metadata, $this->allocator, \FFI::addr($description))); + $this->checkStatus(($this->api->ModelMetadataGetDomain)($metadata, $this->allocator, \FFI::addr($domain))); + $this->checkStatus(($this->api->ModelMetadataGetGraphName)($metadata, $this->allocator, \FFI::addr($graphName))); + $this->checkStatus(($this->api->ModelMetadataGetGraphDescription)($metadata, $this->allocator, \FFI::addr($graphDescription))); + $this->checkStatus(($this->api->ModelMetadataGetProducerName)($metadata, $this->allocator, \FFI::addr($producerName))); + $this->checkStatus(($this->api->ModelMetadataGetVersion)($metadata, \FFI::addr($version))); + + $ret = [ + 'custom_metadata_map' => $customMetadataMap, + 'description' => \FFI::string($description), + 'domain' => \FFI::string($domain), + 'graph_name' => \FFI::string($graphName), + 'graph_description' => \FFI::string($graphDescription), + 'producer_name' => \FFI::string($producerName), + 'version' => $version->cdata + ]; + + // TODO use finally + ($this->api->ReleaseModelMetadata)($metadata); + $this->allocatorFree($description); + $this->allocatorFree($domain); + $this->allocatorFree($graphName); + $this->allocatorFree($graphDescription); + $this->allocatorFree($producerName); + + return $ret; + } + + // return value has double underscore like Python + public function endProfiling() + { + $out = $this->ffi->new('char*'); + $this->checkStatus(($this->api->SessionEndProfiling)($this->session, $this->allocator, \FFI::addr($out))); + return \FFI::string($out); + } + + // no way to set providers with C API yet + // so we can return all available providers + public function providers() + { + $outPtr = $this->ffi->new('char**'); + $lengthPtr = $this->ffi->new('int'); + $this->checkStatus(($this->api->GetAvailableProviders)(\FFI::addr($outPtr), \FFI::addr($lengthPtr))); + $length = $lengthPtr->cdata; + $providers = []; + for ($i = 0; $i < $length; $i++) { + $providers[] = \FFI::string($outPtr[$i]); + } + ($this->api->ReleaseAvailableProviders)($outPtr, $length); + return $providers; + } + + private function loadSession($path, $sessionOptions) + { + $session = $this->ffi->new('OrtSession*'); + if (is_resource($path) && get_resource_type($path) == 'stream') { + $contents = stream_get_contents($path); + $this->checkStatus(($this->api->CreateSessionFromArray)(self::env(), $contents, strlen($contents), $sessionOptions, \FFI::addr($session))); + } else { + $this->checkStatus(($this->api->CreateSession)(self::env(), $this->ortString($path), $sessionOptions, \FFI::addr($session))); + } + return $session; + } + + private function loadAllocator() + { + $allocator = $this->ffi->new('OrtAllocator*'); + $this->checkStatus(($this->api->GetAllocatorWithDefaultOptions)(\FFI::addr($allocator))); + return $allocator; + } + + private function loadInputs() + { + $inputs = []; + $numInputNodes = $this->ffi->new('size_t'); + $this->checkStatus(($this->api->SessionGetInputCount)($this->session, \FFI::addr($numInputNodes))); + for ($i = 0; $i < $numInputNodes->cdata; $i++) { + $namePtr = $this->ffi->new('char*'); + $this->checkStatus(($this->api->SessionGetInputName)($this->session, $i, $this->allocator, \FFI::addr($namePtr))); + // freed in nodeInfo + $typeinfo = $this->ffi->new('OrtTypeInfo*'); + $this->checkStatus(($this->api->SessionGetInputTypeInfo)($this->session, $i, \FFI::addr($typeinfo))); + $inputs[] = array_merge(['name' => \FFI::string($namePtr)], $this->nodeInfo($typeinfo)); + $this->allocatorFree($namePtr); + } + return $inputs; + } + + private function loadOutputs() + { + $outputs = []; + $numOutputNodes = $this->ffi->new('size_t'); + $this->checkStatus(($this->api->SessionGetOutputCount)($this->session, \FFI::addr($numOutputNodes))); + for ($i = 0; $i < $numOutputNodes->cdata; $i++) { + $namePtr = $this->ffi->new('char*'); + $this->checkStatus(($this->api->SessionGetOutputName)($this->session, $i, $this->allocator, \FFI::addr($namePtr))); + // freed in nodeInfo + $typeinfo = $this->ffi->new('OrtTypeInfo*'); + $this->checkStatus(($this->api->SessionGetOutputTypeInfo)($this->session, $i, \FFI::addr($typeinfo))); + $outputs[] = array_merge(['name' => \FFI::string($namePtr)], $this->nodeInfo($typeinfo)); + $this->allocatorFree($namePtr); + } + return $outputs; + } + + private function createInputTensor($inputFeed, &$refs) + { + $allocatorInfo = $this->ffi->new('OrtMemoryInfo*'); + $this->checkStatus(($this->api->CreateCpuMemoryInfo)(1, 0, \FFI::addr($allocatorInfo))); + $inputFeedSize = count($inputFeed); + if ($inputFeedSize == 0) { + throw new Exception('No input'); + } + $inputTensor = $this->ffi->new("OrtValue*[$inputFeedSize]"); + + $idx = 0; + /* @var $input Tensor */ + foreach ($inputFeed as $inputName => $input) { + $inp = null; + foreach ($this->inputs as $i) { + if ($i['name'] == $inputName) { + $inp = $i; + break; + } + } + if (is_null($inp)) { + throw new Exception("Unknown input: $inputName"); + } + + $shape = $input->shape(); + $ndim = $input->ndim(); + $inputNodeDims = $this->ffi->new("int64_t[$ndim]"); + for ($i = 0; $i < $ndim; $i++) { + $inputNodeDims[$i] = $shape[$i]; + } + + if ($inp['type'] == 'tensor(string)') { + $size = $input->size(); + $inputTensorValues = $this->ffi->new("char*[$size]"); + $this->fillStringTensorValues($input, $inputTensorValues, $shape, $refs); + + $typeEnum = $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING; + $this->checkStatus(($this->api->CreateTensorAsOrtValue)($this->allocator, $inputNodeDims, $ndim, $typeEnum, \FFI::addr($inputTensor[$idx]))); + $this->checkStatus(($this->api->FillStringTensor)($inputTensor[$idx], $inputTensorValues, count($inputTensorValues))); + } else { + $size = $input->size(); + + $inputTypes = array_flip(array_map(fn($v) => "tensor($v)", $this->elementDataTypes())); + if (isset($inputTypes[$inp['type']])) { + $typeEnum = $inputTypes[$inp['type']]; + $castType = $this->castTypes()[$typeEnum]; + $inputTensorValues = $this->ffi->new("{$castType}[$size]"); + } else { + $this->unsupportedType('input', $inp['type']); + } + + $this->fillTensorValues($input, $inputTensorValues); + + $this->checkStatus(($this->api->CreateTensorWithDataAsOrtValue)($allocatorInfo, $inputTensorValues, \FFI::sizeof($inputTensorValues), $inputNodeDims, $ndim, $typeEnum, \FFI::addr($inputTensor[$idx]))); + + $refs[] = $inputNodeDims; + $refs[] = $inputTensorValues; + } + $idx++; + } + + // TODO use finally + ($this->api->ReleaseMemoryInfo)($allocatorInfo); + + return $inputTensor; + } + + private function fillStringTensorValues(Tensor $input, $ptr, &$refs): void + { + foreach ($input->buffer() as $i => $v) { + $strPtr = $this->cstring($v); + $ptr[$i] = $strPtr; + $refs[] = $strPtr; + } + } + + private function fillTensorValues(Tensor $input, $ptr): void + { + foreach ($input->buffer() as $i => $v) { + $ptr[$i] = $v; + } + } + + private function createNodeNames($names, &$refs) + { + $namesSize = count($names); + $ptr = $this->ffi->new("char*[$namesSize]"); + foreach ($names as $i => $name) { + $strPtr = $this->cstring($name); + $ptr[$i] = $strPtr; + $refs[] = $strPtr; + } + return $ptr; + } + + private function cstring($str) + { + $bytes = strlen($str) + 1; + // TODO fix? + $ptr = $this->ffi->new("char[$bytes]", owned: false); + \FFI::memcpy($ptr, $str, $bytes - 1); + $ptr[$bytes - 1] = "\0"; + return $ptr; + } + + private function createFromOnnxValue($outPtr) + { + try { + $outType = $this->ffi->new('ONNXType'); + $this->checkStatus(($this->api->GetValueType)($outPtr, \FFI::addr($outType))); + + if ($outType->cdata == $this->ffi->ONNX_TYPE_TENSOR) { + $typeinfo = $this->ffi->new('OrtTensorTypeAndShapeInfo*'); + $this->checkStatus(($this->api->GetTensorTypeAndShape)($outPtr, \FFI::addr($typeinfo))); + + [$type, $shape] = $this->tensorTypeAndShape($typeinfo); + + // TODO skip if string + $tensorData = $this->ffi->new('void*'); + $this->checkStatus(($this->api->GetTensorMutableData)($outPtr, \FFI::addr($tensorData))); + + $outSize = $this->ffi->new('size_t'); + $this->checkStatus(($this->api->GetTensorShapeElementCount)($typeinfo, \FFI::addr($outSize))); + $outputTensorSize = $outSize->cdata; + + ($this->api->ReleaseTensorTypeAndShapeInfo)($typeinfo); + + $castTypes = $this->castTypes(); + if (isset($castTypes[$type])) { + $arr = $this->ffi->cast($castTypes[$type] . "[$outputTensorSize]", $tensorData); + } elseif ($type == $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) { + $arr = $this->createStringsFromOnnxValue($outPtr, $outputTensorSize); + } else { + $this->unsupportedType('element', $type); + } + + return $this->fillOutput($arr, $shape); + } elseif ($outType->cdata == $this->ffi->ONNX_TYPE_SEQUENCE) { + $out = $this->ffi->new('size_t'); + $this->checkStatus(($this->api->GetValueCount)($outPtr, \FFI::addr($out))); + + $ret = []; + for ($i = 0; $i < $out->cdata; $i++) { + $seq = $this->ffi->new('OrtValue*'); + $this->checkStatus(($this->api->GetValue)($outPtr, $i, $this->allocator, \FFI::addr($seq))); + $ret[] = $this->createFromOnnxValue($seq); + } + return $ret; + } elseif ($outType->cdata == $this->ffi->ONNX_TYPE_MAP) { + $typeShape = $this->ffi->new('OrtTensorTypeAndShapeInfo*'); + $mapKeys = $this->ffi->new('OrtValue*'); + $mapValues = $this->ffi->new('OrtValue*'); + $elemType = $this->ffi->new('ONNXTensorElementDataType'); + + $this->checkStatus(($this->api->GetValue)($outPtr, 0, $this->allocator, \FFI::addr($mapKeys))); + $this->checkStatus(($this->api->GetValue)($outPtr, 1, $this->allocator, \FFI::addr($mapValues))); + $this->checkStatus(($this->api->GetTensorTypeAndShape)($mapKeys, \FFI::addr($typeShape))); + $this->checkStatus(($this->api->GetTensorElementType)($typeShape, \FFI::addr($elemType))); + + ($this->api->ReleaseTensorTypeAndShapeInfo)($typeShape); + + // TODO support more types + if ($elemType->cdata == $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) { + $ret = []; + $keys = $this->createFromOnnxValue($mapKeys); + $values = $this->createFromOnnxValue($mapValues); + return array_combine($keys, $values); + } else { + $this->unsupported_type('element', $elemType); + } + } else { + $this->unsupportedType('ONNX', $outType->cdata); + } + } finally { + if (!\FFI::isNull($outPtr)) { + ($this->api->ReleaseValue)($outPtr); + } + } + } + + private function fillOutput($ptr, $shape): ?Tensor + { + $bufferSize = count($ptr); + + if ($bufferSize == 0) { + return null; + } + + $data = []; + + for ($j = 0; $j < $bufferSize; $j++) { + $data[] = $ptr[$j]; + } + + return new Tensor($data, shape: $shape); + } + + private function createStringsFromOnnxValue($outPtr, $outputTensorSize) + { + $len = $this->ffi->new('size_t'); + $this->checkStatus(($this->api->GetStringTensorDataLength)($outPtr, \FFI::addr($len))); + + $sLen = $len->cdata; + $s = $this->ffi->new("char[$sLen]"); + $offsets = $this->ffi->new("size_t[$outputTensorSize]"); + $this->checkStatus(($this->api->GetStringTensorContent)($outPtr, $s, $sLen, $offsets, $outputTensorSize)); + + $result = []; + foreach ($offsets as $i => $v) { + $start = $v; + $end = $i < $outputTensorSize - 1 ? $offsets[$i + 1] : $sLen; + $size = $end - $start; + $result[] = \FFI::string($s + $start, $size); + } + return $result; + } + + private static function checkStatus($status) + { + if (!is_null($status)) { + $message = (self::api()->GetErrorMessage)($status); + (self::api()->ReleaseStatus)($status); + throw new Exception($message); + } + } + + private function nodeInfo($typeinfo) + { + $onnxType = $this->ffi->new('ONNXType'); + $this->checkStatus(($this->api->GetOnnxTypeFromTypeInfo)($typeinfo, \FFI::addr($onnxType))); + + if ($onnxType->cdata == $this->ffi->ONNX_TYPE_TENSOR) { + $tensorInfo = $this->ffi->new('OrtTensorTypeAndShapeInfo*'); + // don't free tensor_info + $this->checkStatus(($this->api->CastTypeInfoToTensorInfo)($typeinfo, \FFI::addr($tensorInfo))); + + [$type, $shape] = $this->tensorTypeAndShape($tensorInfo); + $elementDataType = $this->elementDataTypes()[$type]; + return ['type' => "tensor($elementDataType)", 'shape' => $shape]; + } elseif ($onnxType->cdata == $this->ffi->ONNX_TYPE_SEQUENCE) { + $sequenceTypeInfo = $this->ffi->new('OrtSequenceTypeInfo*'); + $this->checkStatus(($this->api->CastTypeInfoToSequenceTypeInfo)($typeinfo, \FFI::addr($sequenceTypeInfo))); + $nestedTypeInfo = $this->ffi->new('OrtTypeInfo*'); + $this->checkStatus(($this->api->GetSequenceElementType)($sequenceTypeInfo, \FFI::addr($nestedTypeInfo))); + $v = $this->nodeInfo($nestedTypeInfo)['type']; + + return ['type' => "seq($v)", 'shape' => []]; + } elseif ($onnxType->cdata == $this->ffi->ONNX_TYPE_MAP) { + $mapTypeInfo = $this->ffi->new('OrtMapTypeInfo*'); + $this->checkStatus(($this->api->CastTypeInfoToMapTypeInfo)($typeinfo, \FFI::addr($mapTypeInfo))); + + // key + $keyType = $this->ffi->new('ONNXTensorElementDataType'); + $this->checkStatus(($this->api->GetMapKeyType)($mapTypeInfo, \FFI::addr($keyType))); + $k = $this->elementDataTypes()[$keyType->cdata]; + + // value + $valueTypeInfo = $this->ffi->new('OrtTypeInfo*'); + $this->checkStatus(($this->api->GetMapValueType)($mapTypeInfo, \FFI::addr($valueTypeInfo))); + $v = $this->nodeInfo($valueTypeInfo)['type']; + + return ['type' => "map($k,$v)", 'shape' => []]; + } else { + $this->unsupportedType('ONNX', $onnxType->cdata); + } + } + + private function castTypes() + { + return [ + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT => 'float', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 => 'uint8_t', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 => 'int8_t', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 => 'uint16_t', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 => 'int16_t', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 => 'int32_t', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 => 'int64_t', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL => 'bool', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE => 'double', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 => 'uint32_t', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 => 'uint64_t' + ]; + } + + private function elementDataTypes() + { + return [ + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED => 'undefined', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT => 'float', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 => 'uint8', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 => 'int8', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 => 'uint16', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 => 'int16', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 => 'uint32', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 => 'int64', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING => 'string', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL => 'bool', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 => 'float16', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE => 'double', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 => 'uint32', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 => 'uint64', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 => 'complex64', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 => 'complex128', + $this->ffi->ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 => 'bfloat16' + ]; + } + + private function tensorTypeAndShape($tensorInfo) + { + $type = $this->ffi->new('ONNXTensorElementDataType'); + $this->checkStatus(($this->api->GetTensorElementType)($tensorInfo, \FFI::addr($type))); + + $numDimsPtr = $this->ffi->new('size_t'); + $this->checkStatus(($this->api->GetDimensionsCount)($tensorInfo, \FFI::addr($numDimsPtr))); + $numDims = $numDimsPtr->cdata; + + if ($numDims > 0) { + $nodeDims = $this->ffi->new("int64_t[$numDims]"); + $this->checkStatus(($this->api->GetDimensions)($tensorInfo, $nodeDims, $numDims)); + $dims = $this->readArray($nodeDims); + + $symbolicDims = $this->ffi->new("char*[$numDims]"); + $this->checkStatus(($this->api->GetSymbolicDimensions)($tensorInfo, $symbolicDims, $numDims)); + for ($i = 0; $i < $numDims; $i++) { + $namedDim = \FFI::string($symbolicDims[$i]); + if ($namedDim != '') { + $dims[$i] = $namedDim; + } + } + } else { + $dims = []; + } + + return [$type->cdata, $dims]; + } + + private function unsupportedType($name, $type) + { + throw new Exception("Unsupported $name type: $type"); + } + + private function readArray($cdata) + { + $arr = []; + $n = count($cdata); + for ($i = 0; $i < $n; $i++) { + $arr[] = $cdata[$i]; + } + return $arr; + } + + private function allocatorFree($ptr) + { + ($this->api->AllocatorFree)($this->allocator, $ptr); + } + + private static function api() + { + return (FFI::instance()->OrtGetApiBase()[0]->GetApi)(11)[0]; + } + + // wide string on Windows + // char string on Linux + // see ORTCHAR_T in onnxruntime_c_api.h + private function ortString($str) + { + if (PHP_OS_FAMILY == 'Windows') { + $libc = FFI::libc(); + $max = strlen($str) + 1; // for null byte + // wchar_t not supported + // use char instead of casting later + // since FFI::cast only references data + $dest = $libc->new('char[' . ($max * 2) . ']'); + $ret = $libc->mbstowcs($dest, $str, $max); + if ($ret != strlen($str)) { + throw new Exception('Expected mbstowcs to return ' . strlen($str) . ", got $ret"); + } + return $dest; + } else { + return $str; + } + } + + private static function env() + { + // TODO use mutex for thread-safety + // TODO memoize + + $env = FFI::instance()->new('OrtEnv*'); + (self::api()->CreateEnv)(3, 'Default', \FFI::addr($env)); + // disable telemetry + // https://github.com/microsoft/onnxruntime/blob/master/docs/Privacy.md + self::checkStatus((self::api()->DisableTelemetryEvents)($env)); + return $env; + } +} \ No newline at end of file diff --git a/src/Utils/Tensor.php b/src/Utils/Tensor.php index 48edf03..60b6114 100644 --- a/src/Utils/Tensor.php +++ b/src/Utils/Tensor.php @@ -43,10 +43,9 @@ public function __construct($array = null, int $dtype = null, array $shape = nul if (is_array($array) || $array instanceof ArrayObject) { $size = $this->countRecursive($array); $this->buffer = $this->newBuffer($size, $dtype); - $index = 0; - $this->flattenArray($array, $this->buffer, $index); + $this->flattenArray($array, $this->buffer); $this->offset = 0; - $shape = $shape ?? $this->generateShape($array); + $shape ??= $this->generateShape($array); } elseif (is_numeric($array) || is_bool($array)) { if (is_bool($array) && $dtype != NDArray::bool) { throw new InvalidArgumentException("Unmatched dtype with bool value"); @@ -56,7 +55,8 @@ public function __construct($array = null, int $dtype = null, array $shape = nul $this->offset = 0; $shape = $shape ?? []; $this->assertShape($shape); - if (array_product($shape) != 1) + $size = (int)array_product($shape); + if ($size != 1) throw new InvalidArgumentException("Invalid dimension size"); } elseif ($array === null && $shape !== null) { $this->assertShape($shape); @@ -70,19 +70,21 @@ public function __construct($array = null, int $dtype = null, array $shape = nul throw new InvalidArgumentException("Invalid dimension size"); $this->buffer = $array; $this->offset = $offset; + $size = (int)array_product($shape); } else { throw new InvalidArgumentException("Invalid type of array"); } + $this->assertShape($shape); $this->shape = $shape; - $size = (int)array_product($shape); if (count($this->buffer) - $this->offset < $size) throw new InvalidArgumentException("Invalid dimension size"); $this->dtype = $dtype; } + function countRecursive($array): int { $count = 0; @@ -117,35 +119,61 @@ protected function newBuffer(int $size, int $dtype): SplFixedArray|OpenBlasBuffe /** * Flatten the given nested array into a flat array. */ - protected function flattenArray(array|ArrayObject $nestedArray, $flatArray, int &$currentIndex): int - { - $num = null; - $cursor = 0; - $nestedArrayLength = count($nestedArray); + protected function flattenArray(array|ArrayObject $nestedArray, $flatArray, int &$currentIndex = 0): int + { +// $num = null; +// $cursor = 0; +// $nestedArrayLength = count($nestedArray); +// +// while ($cursor < $nestedArrayLength) { +// $value = $nestedArray[$cursor]; +// if (is_array($value) || $value instanceof ArrayObject) { +// if ($value instanceof ArrayObject) { +// $value = $value->getArrayCopy(); +// } +// $num2 = $this->flattenArray($value, $flatArray, $currentIndex); +// if ($num === null) { +// $num = $num2; +// } elseif ($num !== $num2) { +// throw new InvalidArgumentException("The shape of the dimension is broken"); +// } +// } else { +// if ($num !== null) { +// throw new InvalidArgumentException("The shape of the dimension is broken"); +// } +// +// $flatArray[$currentIndex] = $value; +// $currentIndex++; +// } +// $cursor++; +// } +// return $nestedArrayLength; + + + $numElements = 0; + + if ($nestedArray instanceof ArrayObject) { + $nestedArray = $nestedArray->getArrayCopy(); + } - while ($cursor < $nestedArrayLength) { - $value = $nestedArray[$cursor]; + // Iterate through the nested array + foreach ($nestedArray as $value) { + // If the value is an array or ArrayObject, flatten it recursively if (is_array($value) || $value instanceof ArrayObject) { - if ($value instanceof ArrayObject) { - $value = $value->getArrayCopy(); - } - $num2 = $this->flattenArray($value, $flatArray, $currentIndex); - if ($num === null) { - $num = $num2; - } elseif ($num !== $num2) { + $numInNested = $this->flattenArray($value, $flatArray, $currentIndex); + if ($numElements === 0) { + $numElements = $numInNested; + } elseif ($numElements !== $numInNested) { throw new InvalidArgumentException("The shape of the dimension is broken"); } } else { - if ($num !== null) { - throw new InvalidArgumentException("The shape of the dimension is broken"); - } - - $flatArray[$currentIndex] = $value; - $currentIndex++; + // If the value is not an array, append it to the flat array + $flatArray[$currentIndex++] = $value; + $numElements++; } - $cursor++; } - return $nestedArrayLength; + + return $numElements; } /** @@ -155,9 +183,9 @@ protected function generateShape($array): array { $shape = []; - while (is_array($array) || $array instanceof ArrayObject) { + while (is_array($array)) { $shape[] = count($array); - $array = $array[0]; + $array = current($array); } return $shape; From 6c091fbfb6f7f36c4dc9f3f51655b6d2cad1029b Mon Sep 17 00:00:00 2001 From: Kyrian Obikwelu Date: Mon, 22 Apr 2024 17:58:23 +0100 Subject: [PATCH 2/4] Create buffer directly when convert model output to tensor --- examples/pipelines/text-generation.php | 4 ++-- src/Models/Pretrained/PretrainedModel.php | 6 +----- src/Utils/InferenceSession.php | 8 ++++---- src/Utils/Tensor.php | 12 ++++++------ 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/examples/pipelines/text-generation.php b/examples/pipelines/text-generation.php index e8e4ee9..be23f89 100644 --- a/examples/pipelines/text-generation.php +++ b/examples/pipelines/text-generation.php @@ -13,7 +13,7 @@ // //$generator = pipeline('text-generation', 'Xenova/gpt2'); $generator = pipeline('text-generation', 'Xenova/Qwen1.5-0.5B-Chat'); - +// $streamer = StdOutStreamer::make(); $messages = [ @@ -24,7 +24,7 @@ $input = $generator->tokenizer->applyChatTemplate($messages, addGenerationPrompt: true, tokenize: false); $output = $generator($input, -// streamer: $streamer, + streamer: $streamer, maxNewTokens: 128, doSample: true, returnFullText: false, diff --git a/src/Models/Pretrained/PretrainedModel.php b/src/Models/Pretrained/PretrainedModel.php index eb2b93a..2f8249d 100644 --- a/src/Models/Pretrained/PretrainedModel.php +++ b/src/Models/Pretrained/PretrainedModel.php @@ -33,7 +33,6 @@ use Exception; use Symfony\Component\Console\Output\OutputInterface; use function Codewithkyrian\Transformers\Utils\array_some; -use function Codewithkyrian\Transformers\Utils\timeUsage; /** * A base class for pre-trained models that provides the model configuration and an ONNX session. @@ -282,10 +281,7 @@ public function runSession(InferenceSession $session, array $inputs): array $outputNames = array_column($session->outputs(), 'name'); - timeUsage(); - $out = $session->run($outputNames, $inputs); - dump(timeUsage(true)); - return $out; + return $session->run($outputNames, $inputs); } catch (MissingModelInputException $e) { throw $e; } catch (Exception $e) { diff --git a/src/Utils/InferenceSession.php b/src/Utils/InferenceSession.php index 911053d..b82134b 100644 --- a/src/Utils/InferenceSession.php +++ b/src/Utils/InferenceSession.php @@ -130,6 +130,7 @@ public function __destruct() public function run($outputNames, $inputFeed, $logSeverityLevel = null, $logVerbosityLevel = null, $logid = null, $terminate = null): array { + // pointer references $refs = []; @@ -167,7 +168,6 @@ public function run($outputNames, $inputFeed, $logSeverityLevel = null, $logVerb $output = []; foreach ($outputTensor as $i => $t) { -// $output[] = $this->createFromOnnxValue($t); $output[$outputNames[$i]] = $this->createFromOnnxValue($t); } @@ -514,13 +514,13 @@ private function fillOutput($ptr, $shape): ?Tensor return null; } - $data = []; + $buffer = Tensor::newBuffer($bufferSize); for ($j = 0; $j < $bufferSize; $j++) { - $data[] = $ptr[$j]; + $buffer[$j] = $ptr[$j]; } - return new Tensor($data, shape: $shape); + return new Tensor($buffer, shape: $shape, offset: 0); } private function createStringsFromOnnxValue($outPtr, $outputTensorSize) diff --git a/src/Utils/Tensor.php b/src/Utils/Tensor.php index 60b6114..48b3040 100644 --- a/src/Utils/Tensor.php +++ b/src/Utils/Tensor.php @@ -42,7 +42,7 @@ public function __construct($array = null, int $dtype = null, array $shape = nul if (is_array($array) || $array instanceof ArrayObject) { $size = $this->countRecursive($array); - $this->buffer = $this->newBuffer($size, $dtype); + $this->buffer = self::newBuffer($size, $dtype); $this->flattenArray($array, $this->buffer); $this->offset = 0; $shape ??= $this->generateShape($array); @@ -50,7 +50,7 @@ public function __construct($array = null, int $dtype = null, array $shape = nul if (is_bool($array) && $dtype != NDArray::bool) { throw new InvalidArgumentException("Unmatched dtype with bool value"); } - $this->buffer = $this->newBuffer(1, $dtype); + $this->buffer = self::newBuffer(1, $dtype); $this->buffer[0] = $array; $this->offset = 0; $shape = $shape ?? []; @@ -61,7 +61,7 @@ public function __construct($array = null, int $dtype = null, array $shape = nul } elseif ($array === null && $shape !== null) { $this->assertShape($shape); $size = (int)array_product($shape); - $this->buffer = $this->newBuffer($size, $dtype); + $this->buffer = self::newBuffer($size, $dtype); $this->offset = 0; } elseif ($this->isBuffer($array)) { if (!is_int($offset)) @@ -107,7 +107,7 @@ function countRecursive($array): int * @param int $dtype The data type of the buffer. * @return SplFixedArray|OpenBlasBuffer */ - protected function newBuffer(int $size, int $dtype): SplFixedArray|OpenBlasBuffer + public static function newBuffer(int $size, ?int $dtype = null): SplFixedArray|OpenBlasBuffer { if (extension_loaded('rindow_openblas')) { return new OpenBlasBuffer($size, $dtype); @@ -389,7 +389,7 @@ public static function cat(array $tensors, int $axis = 0): Tensor // Create a new array to store the accumulated values $resultSize = array_product($resultShape); - $result = $tensors[0]->newBuffer($resultSize, $resultType); + $result = self::newBuffer($resultSize, $resultType); // Create output tensor of same type as first @@ -921,7 +921,7 @@ public function slice(...$slices): Tensor $newBufferSize = array_reduce($newShape, fn($a, $b) => $a * $b, 1); - $buffer = $this->newBuffer($newBufferSize, $this->dtype()); + $buffer = self::newBuffer($newBufferSize, $this->dtype()); $stride = $this->stride(); for ($i = 0; $i < $newBufferSize; ++$i) { From 0f1c44e2931bfd6efe9c4b2cfb1bc4544e1e2995 Mon Sep 17 00:00:00 2001 From: Kyrian Obikwelu Date: Mon, 22 Apr 2024 23:38:47 +0100 Subject: [PATCH 3/4] Switch TROCR to new inference session --- src/Models/Pretrained/TrOCRPretrainedModel.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Models/Pretrained/TrOCRPretrainedModel.php b/src/Models/Pretrained/TrOCRPretrainedModel.php index 1d8d446..f192db7 100644 --- a/src/Models/Pretrained/TrOCRPretrainedModel.php +++ b/src/Models/Pretrained/TrOCRPretrainedModel.php @@ -8,7 +8,7 @@ use Codewithkyrian\Transformers\Models\ModelArchitecture; use Codewithkyrian\Transformers\Utils\AutoConfig; use Codewithkyrian\Transformers\Utils\GenerationConfig; -use OnnxRuntime\InferenceSession; +use Codewithkyrian\Transformers\Utils\InferenceSession; class TrOCRPretrainedModel extends PretrainedModel { @@ -29,8 +29,8 @@ public function __construct( parent::__construct($config, $session, $modelArchitecture); - $this->numEncoderLayers = $this->numDecoderLayers = $this->config['decoder_layers']; - $this->numEncoderHeads = $this->numDecoderHeads = $this->config['decoder_attention_heads']; - $this->encoderDimKv = $this->decoderDimKv = $this->config['d_model'] / $this->numDecoderHeads; + $this->numEncoderLayers = $this->numDecoderLayers = $this->config['decoder_layers']; + $this->numEncoderHeads = $this->numDecoderHeads = $this->config['decoder_attention_heads']; + $this->encoderDimKv = $this->decoderDimKv = $this->config['d_model'] / $this->numDecoderHeads; } } \ No newline at end of file From 5e310f92ec9d67571e1a9f912c853cd17906d1fc Mon Sep 17 00:00:00 2001 From: Kyrian Obikwelu Date: Tue, 23 Apr 2024 01:00:27 +0100 Subject: [PATCH 4/4] Accept zero in tensor dimensions --- src/Models/ModelArchitecture.php | 8 ------ src/Models/Pretrained/PretrainedModel.php | 34 +++++++++++------------ src/Utils/InferenceSession.php | 8 ++++-- src/Utils/Tensor.php | 2 +- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/Models/ModelArchitecture.php b/src/Models/ModelArchitecture.php index b502330..6ae566b 100644 --- a/src/Models/ModelArchitecture.php +++ b/src/Models/ModelArchitecture.php @@ -220,14 +220,6 @@ protected function decoderForward(PretrainedModel $model, array $modelInputs): a $model->preparePositionIds($inputNames, $decoderFeeds, $useCacheBranch); $model->addPastKeyValues($decoderFeeds, $pastKeyValues); - // The initial past key values should have a shape of 0 in one of the dimensions, which - // is the sequence length. However, I haven't found a way to pass a tensor with a shape of 0 - // to the model, so I'm using a sequence length of 1 instead for the first step, and then - // offsetting the sequence length by 1 for the subsequent steps. This is a workaround for now. - $prevSequenceLength = $decoderFeeds['past_key_values.0.key']->shape()[2]; - $attnMaskLength = $prevSequenceLength == 1 ? 1 : $prevSequenceLength + 1; - $decoderFeeds['attention_mask'] = Tensor::ones([1, $attnMaskLength], dtype: NDArray::int64); - $decoderResults = $model->runSession($model->session, $decoderFeeds); $logits = $decoderResults['logits']; diff --git a/src/Models/Pretrained/PretrainedModel.php b/src/Models/Pretrained/PretrainedModel.php index 2f8249d..278683b 100644 --- a/src/Models/Pretrained/PretrainedModel.php +++ b/src/Models/Pretrained/PretrainedModel.php @@ -467,50 +467,50 @@ public function addPastKeyValues(array &$decoderFeeds, ?array $pastKeyValues): v $decoderFeeds = array_merge($decoderFeeds, $pastKeyValues); } else { // TODO support batches (i.e., batch_size > 1) - $batch_size = 1; + $batchSize = 1; if ($this->config->isEncoderDecoder && ($this->addEncoderPkv ?? true)) { - $encoderShape = [$batch_size, $this->numEncoderHeads, 1, $this->encoderDimKv]; - $decoderShape = [$batch_size, $this->numDecoderHeads, 1, $this->decoderDimKv]; + $encoderShape = [$batchSize, $this->numEncoderHeads, 0, $this->encoderDimKv]; + $decoderShape = [$batchSize, $this->numDecoderHeads, 0, $this->decoderDimKv]; for ($i = 0; $i < $this->numDecoderLayers; ++$i) { $decoderFeeds["past_key_values.$i.encoder.key"] = $decoderFeeds["past_key_values.$i.encoder.value"] - = new Tensor(null, shape: $encoderShape); + = new Tensor([], shape: $encoderShape); $decoderFeeds["past_key_values.$i.decoder.key"] = $decoderFeeds["past_key_values.$i.decoder.value"] - = new Tensor(null, shape: $decoderShape); + = new Tensor([], shape: $decoderShape); } } else if ($this->config->modelType === 'falcon') { // NOTE: Custom implementation for Falcon - $shape = [$batch_size * $this->numHeads, 1, $this->dimKv]; + $shape = [$batchSize * $this->numHeads, 0, $this->dimKv]; for ($i = 0; $i < $this->numLayers; ++$i) { - $decoderFeeds["past_key_values.$i.key"] = new Tensor(null, shape: $shape); - $decoderFeeds["past_key_values.$i.value"] = new Tensor(null, shape: $shape); + $decoderFeeds["past_key_values.$i.key"] = new Tensor([], shape: $shape); + $decoderFeeds["past_key_values.$i.value"] = new Tensor([], shape: $shape); } } else if ($this->config['multi_query'] ?? null) { // e.g., for `gpt_bigcode` - $shape = [$batch_size * $this->numHeads, 1, 2 * $this->dimKv]; + $shape = [$batchSize * $this->numHeads, 0, 2 * $this->dimKv]; for ($i = 0; $i < $this->numLayers; ++$i) { - $decoderFeeds["past_key_values.$i.key_value"] = new Tensor(null, shape: $shape); + $decoderFeeds["past_key_values.$i.key_value"] = new Tensor([], shape: $shape); } } else if ($this->config['model_type'] === 'bloom') { // NOTE: Custom implementation for Bloom - $keyShape = [$batch_size * $this->numHeads, $this->dimKv, 1]; - $valueShape = [$batch_size * $this->numHeads, 1, $this->dimKv]; + $keyShape = [$batchSize * $this->numHeads, $this->dimKv, 0]; + $valueShape = [$batchSize * $this->numHeads, 0, $this->dimKv]; for ($i = 0; $i < $this->numLayers; ++$i) { - $decoderFeeds["past_key_values.$i.key"] = new Tensor(null, shape: $keyShape); - $decoderFeeds["past_key_values.$i.value"] = new Tensor(null, shape: $valueShape); + $decoderFeeds["past_key_values.$i.key"] = new Tensor([], shape: $keyShape); + $decoderFeeds["past_key_values.$i.value"] = new Tensor([], shape: $valueShape); } } else { // Decoder-only - $shape = [$batch_size, $this->numHeads, 1, $this->dimKv]; + $shape = [$batchSize, $this->numHeads, 0, $this->dimKv]; for ($i = 0; $i < $this->numLayers; ++$i) { - $decoderFeeds["past_key_values.$i.key"] = new Tensor(null, shape: $shape); - $decoderFeeds["past_key_values.$i.value"] = new Tensor(null, shape: $shape); + $decoderFeeds["past_key_values.$i.key"] = new Tensor([], shape: $shape); + $decoderFeeds["past_key_values.$i.value"] = new Tensor([], shape: $shape); } } } diff --git a/src/Utils/InferenceSession.php b/src/Utils/InferenceSession.php index b82134b..fae23c1 100644 --- a/src/Utils/InferenceSession.php +++ b/src/Utils/InferenceSession.php @@ -372,7 +372,11 @@ private function createInputTensor($inputFeed, &$refs) if (isset($inputTypes[$inp['type']])) { $typeEnum = $inputTypes[$inp['type']]; $castType = $this->castTypes()[$typeEnum]; - $inputTensorValues = $this->ffi->new("{$castType}[$size]"); + if ($size == 0) { + $inputTensorValues = $this->ffi->new("void *"); + } else { + $inputTensorValues = $this->ffi->new("{$castType}[$size]"); + } } else { $this->unsupportedType('input', $inp['type']); } @@ -494,7 +498,7 @@ private function createFromOnnxValue($outPtr) $values = $this->createFromOnnxValue($mapValues); return array_combine($keys, $values); } else { - $this->unsupported_type('element', $elemType); + $this->unsupportedType('element', $elemType); } } else { $this->unsupportedType('ONNX', $outType->cdata); diff --git a/src/Utils/Tensor.php b/src/Utils/Tensor.php index 48b3040..8468763 100644 --- a/src/Utils/Tensor.php +++ b/src/Utils/Tensor.php @@ -201,7 +201,7 @@ protected function assertShape(array $shape): void throw new InvalidArgumentException( "Invalid shape numbers. It gives " . gettype($num)); } - if ($num <= 0) { + if ($num < 0) { throw new InvalidArgumentException( "Invalid shape numbers. It gives " . $num); }