diff --git a/CMakeLists.txt b/CMakeLists.txt index d58fc9f0a..6eb5dc811 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,6 +117,53 @@ if(NOT _cosmo_nn_backend_count EQUAL 1) message(FATAL_ERROR "Exactly one NN backend must be enabled: Sophon, CPU, or RKNN") endif() +# The selected inference backend is a build-wide compile contract. Business, +# API, and utility OBJECT libraries include the same backend-aware headers as +# the inference layer, so every project target must observe one consistent set +# of feature macros. Chip identity remains separate build data below. +if(COSMO_NN_USE_SOPHON_BACKEND) + add_compile_definitions(COSMO_NN_USE_SOPHON_BACKEND) +elseif(COSMO_NN_USE_CPU_BACKEND) + add_compile_definitions( + COSMO_NN_USE_CPU_BACKEND + COSMO_NN_USE_HOST_BACKEND + COSMO_NN_USE_ONNX_BACKEND) +elseif(COSMO_NN_USE_RKNN_BACKEND) + add_compile_definitions( + COSMO_NN_USE_RKNN_BACKEND + COSMO_NN_USE_HOST_BACKEND + COSMO_NN_USE_RAW_MODEL_BACKEND) +endif() + +# A target package is always chip-specific even when several chips share one +# inference/media implementation. Keep the chip identity as build data instead +# of spreading per-chip preprocessor branches through the RKNN backend. +set(COSMO_TARGET_CHIP "" CACHE STRING "Target accelerator chip recorded in release packages") +if(COSMO_TARGET_CHIP) + string(TOLOWER "${COSMO_TARGET_CHIP}" COSMO_TARGET_CHIP_NORMALIZED) + if(NOT COSMO_TARGET_CHIP_NORMALIZED MATCHES "^(bm1688|cv186x|rk3576|rv1126b|unspecified)$") + message(FATAL_ERROR "Unsupported COSMO_TARGET_CHIP: ${COSMO_TARGET_CHIP}") + endif() +endif() + +if(COSMO_NN_USE_RKNN_BACKEND) + if(NOT COSMO_TARGET_CHIP_NORMALIZED MATCHES "^(rk3576|rv1126b)$") + message(FATAL_ERROR + "RKNN builds require COSMO_TARGET_CHIP=rk3576 or rv1126b") + endif() + string(TOUPPER "${COSMO_TARGET_CHIP_NORMALIZED}" COSMO_TARGET_CHIP_LABEL) + set(COSMO_PLATFORM_PROFILE + "${CMAKE_SOURCE_DIR}/config/rknn/platforms/${COSMO_TARGET_CHIP_NORMALIZED}.json") + if(NOT EXISTS "${COSMO_PLATFORM_PROFILE}") + message(FATAL_ERROR "RKNN platform profile not found: ${COSMO_PLATFORM_PROFILE}") + endif() + add_compile_definitions( + COSMO_RKNN_TARGET_CHIP="${COSMO_TARGET_CHIP_NORMALIZED}" + COSMO_RKNN_TARGET_CHIP_LABEL="${COSMO_TARGET_CHIP_LABEL}") + message(STATUS + "RKNN platform profile: ${COSMO_TARGET_CHIP_NORMALIZED} (${COSMO_PLATFORM_PROFILE})") +endif() + # Media acceleration is selected independently from inference. Defaults keep # existing builds unchanged while allowing a software media pipeline to be # paired with another inference backend. @@ -136,7 +183,7 @@ option(COSMO_MEDIA_USE_SOPHON_BACKEND "Enable Sophon hardware media backend" ${_cosmo_media_sophon_default}) option(COSMO_MEDIA_USE_CPU_BACKEND "Enable FFmpeg software media backend" ${_cosmo_media_cpu_default}) -option(COSMO_MEDIA_USE_ROCKCHIP_BACKEND "Enable Rockchip MPP/RGA Copy-first media backend" +option(COSMO_MEDIA_USE_ROCKCHIP_BACKEND "Enable Rockchip MPP/RGA hardware media backend" ${_cosmo_media_rockchip_default}) set(_cosmo_media_backend_count 0) @@ -778,6 +825,24 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/data/netplan/ ########################################################## # Packaging ########################################################## +if(COSMO_NN_USE_RKNN_BACKEND AND COSMO_PACKAGE_MODELS STREQUAL "include") + if(NOT DEFINED RESOURCE_OVERLAY_DIR OR NOT RESOURCE_OVERLAY_DIR) + message(FATAL_ERROR + "RKNN packages that include models require RESOURCE_OVERLAY_DIR") + endif() + find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_custom_target(verify_rknn_packaging_resources ALL + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_SOURCE_DIR}/tools/rknn/stage_platform_resources.py" + --platform-profile "${COSMO_PLATFORM_PROFILE}" + --output-dir "${RESOURCE_OVERLAY_DIR}" + --verify + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + COMMENT "Verifying target-specific RKNN resource identity and freshness" + VERBATIM + ) +endif() + set(PACKAGE_PREFIX "cosmo") set(PACKAGE_VERSION "V${VER_MAJOR}.${VER_MINOR}.${VER_PATCH}") set(PACKAGE_NAME "${PACKAGE_PREFIX}-${PACKAGE_VERSION}") @@ -786,12 +851,7 @@ set(PACKAGE_NAME "${PACKAGE_PREFIX}-${PACKAGE_VERSION}") # convenient for automation, but it can be separated from the package during a # manual copy. This marker keeps such mistakes observable and prevents packages # for different chips from being byte-identical. -set(COSMO_TARGET_CHIP "" CACHE STRING "Target accelerator chip recorded in release packages") if(COSMO_TARGET_CHIP) - string(TOLOWER "${COSMO_TARGET_CHIP}" COSMO_TARGET_CHIP_NORMALIZED) - if(NOT COSMO_TARGET_CHIP_NORMALIZED MATCHES "^(bm1688|cv186x|rk3576|unspecified)$") - message(FATAL_ERROR "Unsupported COSMO_TARGET_CHIP: ${COSMO_TARGET_CHIP}") - endif() file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/target-chip.txt" "${COSMO_TARGET_CHIP_NORMALIZED}\n") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/target-chip.txt" @@ -800,6 +860,13 @@ if(COSMO_TARGET_CHIP) message(STATUS "Package target chip: ${COSMO_TARGET_CHIP_NORMALIZED}") endif() +if(COSMO_NN_USE_RKNN_BACKEND) + install(FILES "${COSMO_PLATFORM_PROFILE}" + DESTINATION share/cosmo + RENAME platform-profile.json + PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ) +endif() + # Generate and install version.txt file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/version.txt" "${PACKAGE_VERSION}\n") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/version.txt" @@ -868,6 +935,9 @@ set(CPACK_OUTPUT_FILE_PREFIX "${CMAKE_BINARY_DIR}/packages") include(CPack) set(COSMO_PACKAGE_TARGETS ${EXECUTABLE_NAME} web_frontend) +if(TARGET verify_rknn_packaging_resources) + list(APPEND COSMO_PACKAGE_TARGETS verify_rknn_packaging_resources) +endif() add_custom_target(package_all # CPack names are content-addressed only after generation. Clear artifacts # from an earlier profile before creating and exporting this build's output. diff --git a/cmake/rockchip_media.cmake b/cmake/rockchip_media.cmake index fc01ebb1b..3a1cdddfe 100644 --- a/cmake/rockchip_media.cmake +++ b/cmake/rockchip_media.cmake @@ -47,3 +47,11 @@ install(DIRECTORY "${COSMO_ROCKCHIP_MEDIA_ROOT}/lib/" FILES_MATCHING PATTERN "librockchip_mpp.so*" PATTERN "librga.so*") + +set(ROCKCHIP_MEDIA_MANIFEST + "${COSMO_ROCKCHIP_MEDIA_ROOT}/.cosmo-rockchip-media.json") +if(EXISTS "${ROCKCHIP_MEDIA_MANIFEST}") + install(FILES "${ROCKCHIP_MEDIA_MANIFEST}" + DESTINATION share/cosmo/platform + RENAME rockchip-media-manifest.json) +endif() diff --git a/config/rknn/models/helmet.json b/config/rknn/models/helmet.json index 90f325146..0b5dcd04d 100644 --- a/config/rknn/models/helmet.json +++ b/config/rknn/models/helmet.json @@ -4,8 +4,10 @@ "model_type": "classify", "source_repository_path": "data/resource/aiboxresource_x86/models/prod_X86_7982161_helmet_V1.0.0/model.onnx", "source_sha256": "a0ea37d99416371c6ef073d5ac87b486b77bb822b1f8aa29f2f30a8a06c97cdd", + "packaging": { + "algorithm_code": "7982161" + }, "conversion": { - "target_platform": "rk3576", "maximum_onnx_opset": 19, "optimization_level": 3, "preprocessing_owner": "host" @@ -30,6 +32,7 @@ ], "calibration": { "source": "person crops detected in data/test-video/Safety Helmet.mp4", + "person_detector_model": "yolov8", "minimum_samples": 32, "labeled": false }, diff --git a/config/rknn/models/yolov8.json b/config/rknn/models/yolov8.json index 89b1c5480..09432570b 100644 --- a/config/rknn/models/yolov8.json +++ b/config/rknn/models/yolov8.json @@ -4,8 +4,10 @@ "model_type": "yolov8_det", "source_repository_path": "data/resource/aiboxresource_x86/models/prod_X86_9275710_YOLOV8_V1.0.0/model.onnx", "source_sha256": "5f8e91cb4507596b66cd5e3b7193bb16f7bd4779ed3b78e3613fcc51b83db8e1", + "packaging": { + "algorithm_code": "9275710" + }, "conversion": { - "target_platform": "rk3576", "input_sha256": "17a2cd603576afd1b5dd86a485933b857534355d25f517f6e1f2acd2a2fe4563", "maximum_onnx_opset": 19, "maximum_onnx_ir_version": 9, diff --git a/config/rknn/platforms/rk3576.json b/config/rknn/platforms/rk3576.json new file mode 100644 index 000000000..53a08b9a3 --- /dev/null +++ b/config/rknn/platforms/rk3576.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "chip": "rk3576", + "display_name": "RK3576", + "backend": "rknn", + "toolchain_lock": "../toolchain-lock.json", + "conversion": { + "target_platform": "rk3576" + }, + "runtime": { + "architecture": "aarch64", + "default_core_mode": "auto" + }, + "media": { + "default_backend": "rockchip", + "cpu_fallback": true, + "runtime_lock": "../../rockchip-media/runtime-lock.json", + "runtime_profile": "rk3576-build-env-v1", + "require_sealed_sysroot": false + }, + "packaging": { + "directory_token": "RK3576", + "resource_template_directory": "data/resource/aiboxresource_rknn", + "resource_overlay_directory": "data/resource/aiboxresource_rknn", + "legacy_models_directory": "data/resource/aiboxresource_rknn/models" + }, + "device": { + "compatible_tokens": ["rk3576"] + }, + "qualification": { + "status": "candidate-validated", + "requires_target_bound_evidence": true + } +} diff --git a/config/rknn/platforms/rv1126b.json b/config/rknn/platforms/rv1126b.json new file mode 100644 index 000000000..e91246c6f --- /dev/null +++ b/config/rknn/platforms/rv1126b.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "chip": "rv1126b", + "display_name": "RV1126B", + "backend": "rknn", + "toolchain_lock": "../toolchain-lock.json", + "conversion": { + "target_platform": "rv1126b" + }, + "runtime": { + "architecture": "aarch64", + "default_core_mode": "auto" + }, + "media": { + "default_backend": "rockchip", + "cpu_fallback": true, + "runtime_lock": "../../rockchip-media/runtime-lock.json", + "runtime_profile": "mpp-1.1.0-rga-1.10.6", + "require_sealed_sysroot": true + }, + "packaging": { + "directory_token": "RV1126B", + "resource_template_directory": "data/resource/aiboxresource_rknn", + "resource_overlay_directory": "output/platform-artifacts/rv1126b/resource-overlay", + "resource_manifest_required": true, + "legacy_models_directory": "output/platform-artifacts/rv1126b/resource-overlay/models" + }, + "device": { + "compatible_tokens": ["rv1126b"] + }, + "qualification": { + "status": "candidate-validated", + "requires_target_bound_evidence": true + } +} diff --git a/config/rknn/toolchain-lock.json b/config/rknn/toolchain-lock.json index 5decdd9cd..4e1b982a1 100644 --- a/config/rknn/toolchain-lock.json +++ b/config/rknn/toolchain-lock.json @@ -1,19 +1,21 @@ { - "schema_version": 1, - "target_platform": "rk3576", - "cosmo_edge_base_sha": "2eaf5fd7b096f98a9dca1ef298e03440484e15bc", - "tensorrt_reference_sha": "57c578616ba826f01921bb4e43d7a695549b6b9e", - "rknn_toolkit2": { - "version": "2.3.2", - "tag_sha": "42aa1d426c0a9e0869b6374edba009f7208a1926", - "python": "3.10", - "wheel_sha256": "6cb783ddf293ac509f39bf9127acf6a5492bbb67e4b4b4ac33a7c6d2cefb4f3c" + "schema_version": 2, + "family": "rknn-toolkit2", + "version": "2.3.2", + "python": "3.10", + "source": { + "repository": "https://github.com/airockchip/rknn-toolkit2", + "tag_sha": "42aa1d426c0a9e0869b6374edba009f7208a1926" + }, + "wheel": { + "distribution": "rknn-toolkit2", + "sha256": "6cb783ddf293ac509f39bf9127acf6a5492bbb67e4b4b4ac33a7c6d2cefb4f3c" }, - "rknn_model_zoo": { + "model_zoo": { "version": "2.3.2", "tag_sha": "bad6c7334531becaf90a561988519b7bec34d0ab" }, - "offline_bundle": { + "known_offline_bundle": { "file": "rk3576-offline-bundle-v2.3.2-20260804.tar.gz", "sha256": "12960656f854ab8bfe0e2ee2292745245d82d1cd88b7acd0a8cb6c06c080c726", "entry_count": 42 @@ -24,12 +26,6 @@ "rknn_api_header_sha256": "c48e11a6f41b451a5fd1e4ad774ea60252d3d94f78bee9b21ea3d21b21deba9a", "rknn_server_sha256": "eea12fe4270fad8aff015056319705b2eb871563ebd001eff8d8788bdd1c0cfa" }, - "device_baseline": { - "kernel": "6.1.118", - "rknpu_driver": "0.9.8", - "system_runtime": "2.1.0", - "system_runtime_sha256": "0f1c5db6c649c8705759308aa56ecffc3295c7eb465f9ec9170882f44004aa8d" - }, "models": { "helmet": { "onnx_sha256": "a0ea37d99416371c6ef073d5ac87b486b77bb822b1f8aa29f2f30a8a06c97cdd", @@ -44,5 +40,22 @@ "input": [1, 3, 640, 640], "output": [1, 84, 8400] } + }, + "qualification": { + "rk3576": { + "status": "candidate-validated", + "cosmo_edge_base_sha": "2eaf5fd7b096f98a9dca1ef298e03440484e15bc", + "tensorrt_reference_sha": "57c578616ba826f01921bb4e43d7a695549b6b9e", + "device_baseline": { + "kernel": "6.1.118", + "rknpu_driver": "0.9.8", + "system_runtime": "2.1.0", + "system_runtime_sha256": "0f1c5db6c649c8705759308aa56ecffc3295c7eb465f9ec9170882f44004aa8d" + } + }, + "rv1126b": { + "status": "candidate-validated", + "requires_target_bound_evidence": true + } } } diff --git a/config/rockchip-media/runtime-lock.json b/config/rockchip-media/runtime-lock.json new file mode 100644 index 000000000..02f71579d --- /dev/null +++ b/config/rockchip-media/runtime-lock.json @@ -0,0 +1,78 @@ +{ + "schema_version": 1, + "family": "rockchip-mpp-rga", + "runtimes": { + "rk3576-build-env-v1": { + "architecture": "aarch64", + "provenance": "legacy-build-image", + "sources": { + "builder": { + "repository": "ghcr.io/cosmo-wander-ai/cosmo_edge-build-env_rk3576", + "revision": "sha256:135d25d0baf14e7918726f7efb040a0627926aedd5825f52fab6c1cd208da348" + } + }, + "artifacts": { + "include/rockchip/rk_mpi.h": {}, + "include/rga/im2d.h": {}, + "lib/librockchip_mpp.so": { + "elf": { + "machine": "AArch64", + "soname": "librockchip_mpp.so.1" + } + }, + "lib/librga.so": { + "elf": { + "machine": "AArch64", + "soname": "librga.so.2" + } + } + } + }, + "mpp-1.1.0-rga-1.10.6": { + "architecture": "aarch64", + "provenance": "source-and-official-prebuilt", + "sources": { + "mpp": { + "repository": "https://github.com/rockchip-linux/mpp", + "revision": "c08762ebfadeb4e986d2fed993bc7a54862d3ebe", + "tag": "1.1.0" + }, + "rga": { + "repository": "https://github.com/airockchip/librga", + "revision": "2b32edcb97b601b25683e2941d888c8515da6d55", + "api_version": "1.10.6_[3]", + "artifact": "libs/Linux/gcc-aarch64/librga.so" + } + }, + "artifacts": { + "include/rockchip/rk_mpi.h": { + "sha256": "96dbae7d90e91baaac38821fc06c74a327cac888007841fe5937798259193646" + }, + "include/rga/im2d.h": { + "sha256": "5ee06891775451c56e6264f5d53db0a4e948e1830287f3a5db4f75c30ac0ff11" + }, + "include/rga/im2d_version.h": { + "sha256": "66021285feb75dc1b492345adc074722813b8945dd998357002cf5396872980c" + }, + "lib/librockchip_mpp.so.0": { + "sha256": "c27de803917989f85c1435261ddcf4db16182bd72a6830090e98031e89700e0e", + "elf": { + "machine": "AArch64", + "soname": "librockchip_mpp.so.1" + } + }, + "lib/librga.so": { + "sha256": "e150bda757fb5e8a649c429ec7cabaf851aa2a3be554ed494d5519e8790d943b", + "elf": { + "machine": "AArch64", + "soname": "librga.so" + } + } + }, + "links": { + "lib/librockchip_mpp.so": "librockchip_mpp.so.1", + "lib/librockchip_mpp.so.1": "librockchip_mpp.so.0" + } + } + } +} diff --git a/data/resource/aiboxresource_rknn/models/prod_RK3576_7982161_helmet_V1.0.0/config.json b/data/resource/aiboxresource_rknn/models/prod_RK3576_7982161_helmet_V1.0.0/config.json index 632695291..312fdbb3e 100644 --- a/data/resource/aiboxresource_rknn/models/prod_RK3576_7982161_helmet_V1.0.0/config.json +++ b/data/resource/aiboxresource_rknn/models/prod_RK3576_7982161_helmet_V1.0.0/config.json @@ -1 +1 @@ -{"algorithm_code":"7982161","chip_type":"RK3576","labels":[{"id":"0","name":"helmet","threshold":[0.5,0.5]},{"id":"1","name":"nohelmet","threshold":[0.5,0.5]}],"model_type":"classify","models":[{"description":"","file_md5":"","file_name":"","inputs":[{"data_type":0,"name":"images","shape":[1,3,224,224]}],"max_batch":1,"name":"helmet","outputs":[{"data_type":0,"name":"output0","shape":[1,2]}],"params":{"crop":true,"crop_h_bottom":0,"crop_h_top":0,"crop_w_left":0,"crop_w_right":0,"gravity":0,"input_size":[224,224],"is_bgr":false,"normalize_mean":[0,0,0],"normalize_scale":0.00392157,"padding_color":[114,114,114],"square":false,"square_mode":0}}],"reduce":"","version":"V1.0.0"} \ No newline at end of file +{"algorithm_code":"7982161","chip_type":"RK3576","labels":[{"id":"0","name":"helmet","threshold":[0.5,0.5]},{"id":"1","name":"nohelmet","threshold":[0.5,0.5]}],"model_type":"classify","models":[{"description":"","file_md5":"","file_name":"","inputs":[{"data_type":0,"name":"images","shape":[1,3,224,224]}],"max_batch":1,"name":"helmet","outputs":[{"data_type":0,"name":"output0","shape":[1,2]}],"params":{"crop":true,"crop_h_bottom":0,"crop_h_top":0,"crop_w_left":0,"crop_w_right":0,"gravity":0,"input_size":[224,224],"is_bgr":false,"normalize_mean":[0,0,0],"normalize_scale":0.00392157,"padding_color":[114,114,114],"rknn_input_contract":"cosmo.rknn.input.rgb_u8_nhwc_0_255_to_0_1.v1","square":false,"square_mode":0}}],"reduce":"","version":"V1.0.0"} diff --git a/data/resource/aiboxresource_rknn/models/prod_RK3576_9275710_YOLOV8_V1.0.0/config.json b/data/resource/aiboxresource_rknn/models/prod_RK3576_9275710_YOLOV8_V1.0.0/config.json index f04d58bd4..ee5a97dcc 100644 --- a/data/resource/aiboxresource_rknn/models/prod_RK3576_9275710_YOLOV8_V1.0.0/config.json +++ b/data/resource/aiboxresource_rknn/models/prod_RK3576_9275710_YOLOV8_V1.0.0/config.json @@ -1 +1 @@ -{"algorithm_code":"9275710","chip_type":"RK3576","labels":[{"id":"0","name":"Pedestrian","threshold":[0.25,0.25]},{"id":"1","name":"category1","threshold":[0.25,0.25]},{"id":"2","name":"category2","threshold":[0.25,0.25]},{"id":"3","name":"category3","threshold":[0.25,0.25]},{"id":"4","name":"category4","threshold":[0.25,0.25]},{"id":"5","name":"category5","threshold":[0.25,0.25]},{"id":"6","name":"category6","threshold":[0.25,0.25]},{"id":"7","name":"category7","threshold":[0.25,0.25]},{"id":"8","name":"category8","threshold":[0.25,0.25]},{"id":"9","name":"category9","threshold":[0.25,0.25]},{"id":"10","name":"category10","threshold":[0.25,0.25]},{"id":"11","name":"category11","threshold":[0.25,0.25]},{"id":"12","name":"category12","threshold":[0.25,0.25]},{"id":"13","name":"category13","threshold":[0.25,0.25]},{"id":"14","name":"category14","threshold":[0.25,0.25]},{"id":"15","name":"category15","threshold":[0.25,0.25]},{"id":"16","name":"category16","threshold":[0.25,0.25]},{"id":"17","name":"category17","threshold":[0.25,0.25]},{"id":"18","name":"category18","threshold":[0.25,0.25]},{"id":"19","name":"category19","threshold":[0.25,0.25]},{"id":"20","name":"category20","threshold":[0.25,0.25]},{"id":"21","name":"category21","threshold":[0.25,0.25]},{"id":"22","name":"category22","threshold":[0.25,0.25]},{"id":"23","name":"category23","threshold":[0.25,0.25]},{"id":"24","name":"category24","threshold":[0.25,0.25]},{"id":"25","name":"category25","threshold":[0.25,0.25]},{"id":"26","name":"category26","threshold":[0.25,0.25]},{"id":"27","name":"category27","threshold":[0.25,0.25]},{"id":"28","name":"category28","threshold":[0.25,0.25]},{"id":"29","name":"category29","threshold":[0.25,0.25]},{"id":"30","name":"category30","threshold":[0.25,0.25]},{"id":"31","name":"category31","threshold":[0.25,0.25]},{"id":"32","name":"category32","threshold":[0.25,0.25]},{"id":"33","name":"category33","threshold":[0.25,0.25]},{"id":"34","name":"category34","threshold":[0.25,0.25]},{"id":"35","name":"category35","threshold":[0.25,0.25]},{"id":"36","name":"category36","threshold":[0.25,0.25]},{"id":"37","name":"category37","threshold":[0.25,0.25]},{"id":"38","name":"category38","threshold":[0.25,0.25]},{"id":"39","name":"category39","threshold":[0.25,0.25]},{"id":"40","name":"category40","threshold":[0.25,0.25]},{"id":"41","name":"category41","threshold":[0.25,0.25]},{"id":"42","name":"category42","threshold":[0.25,0.25]},{"id":"43","name":"category43","threshold":[0.25,0.25]},{"id":"44","name":"category44","threshold":[0.25,0.25]},{"id":"45","name":"category45","threshold":[0.25,0.25]},{"id":"46","name":"category46","threshold":[0.25,0.25]},{"id":"47","name":"category47","threshold":[0.25,0.25]},{"id":"48","name":"category48","threshold":[0.25,0.25]},{"id":"49","name":"category49","threshold":[0.25,0.25]},{"id":"50","name":"category50","threshold":[0.25,0.25]},{"id":"51","name":"category51","threshold":[0.25,0.25]},{"id":"52","name":"category52","threshold":[0.25,0.25]},{"id":"53","name":"category53","threshold":[0.25,0.25]},{"id":"54","name":"category54","threshold":[0.25,0.25]},{"id":"55","name":"category55","threshold":[0.25,0.25]},{"id":"56","name":"category56","threshold":[0.25,0.25]},{"id":"57","name":"category57","threshold":[0.25,0.25]},{"id":"58","name":"category58","threshold":[0.25,0.25]},{"id":"59","name":"category59","threshold":[0.25,0.25]},{"id":"60","name":"category60","threshold":[0.25,0.25]},{"id":"61","name":"category61","threshold":[0.25,0.25]},{"id":"62","name":"category62","threshold":[0.25,0.25]},{"id":"63","name":"category63","threshold":[0.25,0.25]},{"id":"64","name":"category64","threshold":[0.25,0.25]},{"id":"65","name":"category65","threshold":[0.25,0.25]},{"id":"66","name":"category66","threshold":[0.25,0.25]},{"id":"67","name":"category67","threshold":[0.25,0.25]},{"id":"68","name":"category68","threshold":[0.25,0.25]},{"id":"69","name":"category69","threshold":[0.25,0.25]},{"id":"70","name":"category70","threshold":[0.25,0.25]},{"id":"71","name":"category71","threshold":[0.25,0.25]},{"id":"72","name":"category72","threshold":[0.25,0.25]},{"id":"73","name":"category73","threshold":[0.25,0.25]},{"id":"74","name":"category74","threshold":[0.25,0.25]},{"id":"75","name":"category75","threshold":[0.25,0.25]},{"id":"76","name":"category76","threshold":[0.25,0.25]},{"id":"77","name":"category77","threshold":[0.25,0.25]},{"id":"78","name":"category78","threshold":[0.25,0.25]},{"id":"79","name":"category79","threshold":[0.25,0.25]}],"model_type":"yolov8_det","models":[{"description":"YOLOV8检测","file_md5":"","file_name":"","inputs":[{"data_type":0,"name":"images","shape":[1,3,640,640]}],"max_batch":1,"name":"YOLOV8","outputs":[{"data_type":0,"name":"output0","shape":[1,84,8400]}],"params":{"confidence_threshold":0.25,"gravity":1,"input_size":[640,640],"is_bgr":false,"nms_threshold":0.7,"normalize_mean":[0,0,0],"normalize_scale":0.00392157,"padding_color":[114,114,114],"top_k":1000}}],"reduce":"","version":"V1.0.0"} \ No newline at end of file +{"algorithm_code":"9275710","chip_type":"RK3576","labels":[{"id":"0","name":"Pedestrian","threshold":[0.25,0.25]},{"id":"1","name":"category1","threshold":[0.25,0.25]},{"id":"2","name":"category2","threshold":[0.25,0.25]},{"id":"3","name":"category3","threshold":[0.25,0.25]},{"id":"4","name":"category4","threshold":[0.25,0.25]},{"id":"5","name":"category5","threshold":[0.25,0.25]},{"id":"6","name":"category6","threshold":[0.25,0.25]},{"id":"7","name":"category7","threshold":[0.25,0.25]},{"id":"8","name":"category8","threshold":[0.25,0.25]},{"id":"9","name":"category9","threshold":[0.25,0.25]},{"id":"10","name":"category10","threshold":[0.25,0.25]},{"id":"11","name":"category11","threshold":[0.25,0.25]},{"id":"12","name":"category12","threshold":[0.25,0.25]},{"id":"13","name":"category13","threshold":[0.25,0.25]},{"id":"14","name":"category14","threshold":[0.25,0.25]},{"id":"15","name":"category15","threshold":[0.25,0.25]},{"id":"16","name":"category16","threshold":[0.25,0.25]},{"id":"17","name":"category17","threshold":[0.25,0.25]},{"id":"18","name":"category18","threshold":[0.25,0.25]},{"id":"19","name":"category19","threshold":[0.25,0.25]},{"id":"20","name":"category20","threshold":[0.25,0.25]},{"id":"21","name":"category21","threshold":[0.25,0.25]},{"id":"22","name":"category22","threshold":[0.25,0.25]},{"id":"23","name":"category23","threshold":[0.25,0.25]},{"id":"24","name":"category24","threshold":[0.25,0.25]},{"id":"25","name":"category25","threshold":[0.25,0.25]},{"id":"26","name":"category26","threshold":[0.25,0.25]},{"id":"27","name":"category27","threshold":[0.25,0.25]},{"id":"28","name":"category28","threshold":[0.25,0.25]},{"id":"29","name":"category29","threshold":[0.25,0.25]},{"id":"30","name":"category30","threshold":[0.25,0.25]},{"id":"31","name":"category31","threshold":[0.25,0.25]},{"id":"32","name":"category32","threshold":[0.25,0.25]},{"id":"33","name":"category33","threshold":[0.25,0.25]},{"id":"34","name":"category34","threshold":[0.25,0.25]},{"id":"35","name":"category35","threshold":[0.25,0.25]},{"id":"36","name":"category36","threshold":[0.25,0.25]},{"id":"37","name":"category37","threshold":[0.25,0.25]},{"id":"38","name":"category38","threshold":[0.25,0.25]},{"id":"39","name":"category39","threshold":[0.25,0.25]},{"id":"40","name":"category40","threshold":[0.25,0.25]},{"id":"41","name":"category41","threshold":[0.25,0.25]},{"id":"42","name":"category42","threshold":[0.25,0.25]},{"id":"43","name":"category43","threshold":[0.25,0.25]},{"id":"44","name":"category44","threshold":[0.25,0.25]},{"id":"45","name":"category45","threshold":[0.25,0.25]},{"id":"46","name":"category46","threshold":[0.25,0.25]},{"id":"47","name":"category47","threshold":[0.25,0.25]},{"id":"48","name":"category48","threshold":[0.25,0.25]},{"id":"49","name":"category49","threshold":[0.25,0.25]},{"id":"50","name":"category50","threshold":[0.25,0.25]},{"id":"51","name":"category51","threshold":[0.25,0.25]},{"id":"52","name":"category52","threshold":[0.25,0.25]},{"id":"53","name":"category53","threshold":[0.25,0.25]},{"id":"54","name":"category54","threshold":[0.25,0.25]},{"id":"55","name":"category55","threshold":[0.25,0.25]},{"id":"56","name":"category56","threshold":[0.25,0.25]},{"id":"57","name":"category57","threshold":[0.25,0.25]},{"id":"58","name":"category58","threshold":[0.25,0.25]},{"id":"59","name":"category59","threshold":[0.25,0.25]},{"id":"60","name":"category60","threshold":[0.25,0.25]},{"id":"61","name":"category61","threshold":[0.25,0.25]},{"id":"62","name":"category62","threshold":[0.25,0.25]},{"id":"63","name":"category63","threshold":[0.25,0.25]},{"id":"64","name":"category64","threshold":[0.25,0.25]},{"id":"65","name":"category65","threshold":[0.25,0.25]},{"id":"66","name":"category66","threshold":[0.25,0.25]},{"id":"67","name":"category67","threshold":[0.25,0.25]},{"id":"68","name":"category68","threshold":[0.25,0.25]},{"id":"69","name":"category69","threshold":[0.25,0.25]},{"id":"70","name":"category70","threshold":[0.25,0.25]},{"id":"71","name":"category71","threshold":[0.25,0.25]},{"id":"72","name":"category72","threshold":[0.25,0.25]},{"id":"73","name":"category73","threshold":[0.25,0.25]},{"id":"74","name":"category74","threshold":[0.25,0.25]},{"id":"75","name":"category75","threshold":[0.25,0.25]},{"id":"76","name":"category76","threshold":[0.25,0.25]},{"id":"77","name":"category77","threshold":[0.25,0.25]},{"id":"78","name":"category78","threshold":[0.25,0.25]},{"id":"79","name":"category79","threshold":[0.25,0.25]}],"model_type":"yolov8_det","models":[{"description":"YOLOV8检测","file_md5":"","file_name":"","inputs":[{"data_type":0,"name":"images","shape":[1,3,640,640]}],"max_batch":1,"name":"YOLOV8","outputs":[{"data_type":0,"name":"output0","shape":[1,84,8400]}],"params":{"confidence_threshold":0.25,"gravity":1,"input_size":[640,640],"is_bgr":false,"nms_threshold":0.7,"normalize_mean":[0,0,0],"normalize_scale":0.00392157,"padding_color":[114,114,114],"rknn_input_contract":"cosmo.rknn.input.rgb_u8_nhwc_0_255_to_0_1.v1","top_k":1000}}],"reduce":"","version":"V1.0.0"} diff --git a/docs/en/guide/rk3576-rknn-development.md b/docs/en/guide/rk3576-rknn-development.md index 42d4198f5..2c403f9d2 100644 --- a/docs/en/guide/rk3576-rknn-development.md +++ b/docs/en/guide/rk3576-rknn-development.md @@ -123,7 +123,9 @@ python tools/rknn/prepare_validation_data.py \ --output-dir yolov8-calibration --samples 32 python tools/rknn/convert_model.py \ - --spec config/rknn/models/yolov8.json --model yolov8-heads.onnx \ + --spec config/rknn/models/yolov8.json \ + --platform-profile config/rknn/platforms/rk3576.json \ + --model yolov8-heads.onnx \ --output yolov8-heads-int8.rknn --quantize \ --dataset yolov8-calibration/dataset.txt ``` diff --git a/docs/en/tutorials/05-model-porting/model-porting.md b/docs/en/tutorials/05-model-porting/model-porting.md index 79180b3b3..ffb5f9f46 100644 --- a/docs/en/tutorials/05-model-porting/model-porting.md +++ b/docs/en/tutorials/05-model-porting/model-porting.md @@ -14,14 +14,14 @@ next: false | Who this is for | ML engineers and integration developers bringing a custom detector or classifier to CosmoEdge | | What you will accomplish | Evaluate runtime compatibility, convert and upload a model, configure parsing, and complete image, video, and sustained-run validation | | Prerequisites | Understand Pipelines and know the model input, output, preprocessing, postprocessing, and label order | -| Estimated time | About 40–60 minutes for x86 ONNX; Sophon conversion commonly adds 30–60 minutes | -| Device required | x86 requires an ONNX Runtime CosmoEdge build; Sophon requires a BM1688/CV186X device and matching conversion toolchain | +| Estimated time | About 40–60 minutes for x86 ONNX; Sophon or Rockchip conversion commonly adds 30–60 minutes | +| Device required | x86 requires an ONNX Runtime CosmoEdge build; Sophon and Rockchip require the actual target device and matching conversion toolchain | | Final acceptance result | The model loads, its output is parsed correctly, image and video results pass, and it runs without resource failure on the target device | Complete third-party integration in this order: 1. Confirm support conditions and the model contract. -2. Export ONNX; for Sophon, convert it again into a chip-specific `bmodel`. +2. Export ONNX; convert it into a chip-specific `bmodel` for Sophon or `rknn` for Rockchip. 3. Validate the artifact on the conversion host. 4. Upload and configure the model. 5. Run positive and negative image tests first. @@ -39,12 +39,14 @@ output layout, and postprocessing are compatible with CosmoEdge. | --- | --- | --- | --- | --- | | x86 CPU | `.onnx` | `model.onnx` | ONNX Runtime CPU | x86_64 host and matching CosmoEdge build | | Sophon | `.bmodel` | `model.nn` | Sophon BMRT | BM1688 or CV186X; the artifact must target the actual chip | +| Rockchip RKNN | `.rknn` | `model.rknn` | RKNN Runtime | RK3576 or RV1126B; the artifact must target the actual chip | `model.nn` is the internal file name in a CosmoEdge model package. It wraps the device model. When adding an individual Sophon model in the UI, select its `.bmodel`; do not rename an extension to `.nn`. PyTorch `.pt`, TensorFlow SavedModel, and other training-framework artifacts cannot be uploaded directly. -Export them to ONNX first. Sophon deployments then convert ONNX into a chip-specific `.bmodel`. +Export them to ONNX first. Sophon deployments then convert ONNX into a chip-specific `.bmodel`, while +Rockchip deployments produce a chip-specific `.rknn`. RK3576 and RV1126B `.rknn` artifacts are not interchangeable. ### 1.2 Contracts Beyond the File Format @@ -66,7 +68,7 @@ or runtime code. ### 1.3 Verified Capability vs Conditional Compatibility - **Directly supported by current code**: Add `.onnx` on x86, add `.bmodel` on Sophon, and import packages - containing `model.onnx` or `model.nn`. + containing `model.onnx` or `model.nn`; RKNN builds add `.rknn` and package it as `model.rknn`. - **Reference evidence in this repository**: a YOLOv8 detector has completed x86 ONNX import, live overlay, and event output. - **Still required on the target candidate**: validate your exact model, Sophon artifact, performance, @@ -387,6 +389,32 @@ task acceptance, but it must not borrow another example's fixed shapes or hashes The Sophon Add Model page requires a `.bmodel` file. +## Rockchip RKNN Path: Shared Backend, Target-Specific Artifacts + +RK3576 and RV1126B share one CosmoEdge RKNN inference implementation, Rockchip media interface, and model +contract. Chip differences come from `config/rknn/platforms/.json`. A model spec does not hard-code a +chip, but every conversion binds one platform profile, so the resulting `.rknn` remains target-specific and +must not be copied between chips. + +The agent-assisted flow always enters through `scripts/agent/convert_model.sh` and `verify.sh`. It selects +RKNN Toolkit2 from the task contract and freezes Python, wheel, platform profile, model spec, calibration +set, and artifact hashes. RK3576 and RV1126B do not maintain separate conversion scripts. For manual +diagnosis, follow the [RK3576 RKNN development guide](/en/guide/rk3576-rknn-development) and select the +actual target profile: + +```bash +python tools/rknn/convert_model.py \ + --spec config/rknn/models/yolov8.json \ + --platform-profile config/rknn/platforms/rv1126b.json \ + --model yolov8-heads.onnx \ + --output yolov8-rv1126b-int8.rknn \ + --quantize --dataset calibration/dataset.txt +``` + +Conversion-host success is not device acceptance. Validate the matching Runtime/driver, numerical output, +image and video postprocessing, OSD, rules, alerts, the 5 FPS target, and stability on the actual device, +with results bound to the target chip and artifact SHA-256. + ## 4. Upload and Configure the Model ### 4.1 Open Model Repository diff --git a/docs/guide/rk3576-rknn-development.md b/docs/guide/rk3576-rknn-development.md index 329260155..82730e4cd 100644 --- a/docs/guide/rk3576-rknn-development.md +++ b/docs/guide/rk3576-rknn-development.md @@ -107,7 +107,9 @@ python tools/rknn/prepare_validation_data.py \ --output-dir yolov8-calibration --samples 32 python tools/rknn/convert_model.py \ - --spec config/rknn/models/yolov8.json --model yolov8-heads.onnx \ + --spec config/rknn/models/yolov8.json \ + --platform-profile config/rknn/platforms/rk3576.json \ + --model yolov8-heads.onnx \ --output yolov8-heads-int8.rknn --quantize \ --dataset yolov8-calibration/dataset.txt ``` diff --git a/docs/tutorials/05-model-porting/model-porting.md b/docs/tutorials/05-model-porting/model-porting.md index 91e7e006b..7fa11c7f5 100644 --- a/docs/tutorials/05-model-porting/model-porting.md +++ b/docs/tutorials/05-model-porting/model-porting.md @@ -14,14 +14,14 @@ next: false | 适合谁 | 需要把自有检测或分类模型接入 CosmoEdge 的算法工程师和集成开发者 | | 完成后能做什么 | 判断模型是否满足运行条件，转换并上传模型，配置解析参数，完成图片、视频和持续运行验证 | | 使用前提 | 已理解 Pipeline；掌握模型输入、输出、预处理、后处理和标签顺序 | -| 预计时间 | x86 ONNX 路径约 40–60 分钟；Sophon 转换路径通常需要额外 30–60 分钟 | -| 是否需要设备 | x86 路径需要 ONNX Runtime 版 CosmoEdge；Sophon 路径需要 BM1688/CV186X 设备及匹配转换工具链 | +| 预计时间 | x86 ONNX 路径约 40–60 分钟；Sophon 或 Rockchip 转换路径通常需要额外 30–60 分钟 | +| 是否需要设备 | x86 路径需要 ONNX Runtime 版 CosmoEdge；Sophon 与 Rockchip 路径需要目标芯片设备及匹配转换工具链 | | 最终验收结果 | 模型可加载、推理输出可解析、图片与视频结果正确，并在目标设备上持续运行无资源错误 | 第三方模型接入按以下顺序完成： 1. 确认支持条件和模型契约。 -2. 导出 ONNX；Sophon 设备再转换为目标芯片的 `bmodel`。 +2. 导出 ONNX；Sophon 设备转换为目标芯片的 `bmodel`，Rockchip 设备转换为目标芯片的 `rknn`。 3. 在转换主机上检查产物。 4. 上传并配置模型。 5. 先做正负图片验证。 @@ -39,12 +39,14 @@ next: false | --- | --- | --- | --- | --- | | x86 CPU | `.onnx` | `model.onnx` | ONNX Runtime CPU | x86_64 主机和对应 CosmoEdge 构建 | | Sophon | `.bmodel` | `model.nn` | Sophon BMRT | BM1688 或 CV186X，转换产物必须匹配芯片 | +| Rockchip RKNN | `.rknn` | `model.rknn` | RKNN Runtime | RK3576 或 RV1126B，转换产物必须匹配实际芯片 | `model.nn` 是 CosmoEdge 模型包中的内部文件名，封装的是设备侧模型；通过页面单独添加 Sophon 模型时应选择 `.bmodel`，不要把文件扩展名手工改成 `.nn`。 PyTorch `.pt`、TensorFlow SavedModel 或其他训练框架产物不能直接上传。它们必须先导出 -为 ONNX；Sophon 还要使用匹配工具链把 ONNX 转为目标芯片的 `.bmodel`。 +为 ONNX；Sophon 还要使用匹配工具链把 ONNX 转为目标芯片的 `.bmodel`，Rockchip 则转换 +为目标芯片的 `.rknn`。RK3576 与 RV1126B 的 `.rknn` 不是可互换模型。 ### 1.2 格式之外还要匹配的契约 @@ -65,7 +67,7 @@ CosmoEdge 当前已有 `YOLOV8_DET` 等解析路径，但“任意 ONNX”并不 ### 1.3 已验证能力与条件性兼容 - **由当前代码直接支持**：x86 添加 `.onnx`、Sophon 添加 `.bmodel`，以及模型包中的 - `model.onnx` / `model.nn`。 + `model.onnx` / `model.nn`；RKNN 构建添加 `.rknn`，模型包中使用 `model.rknn`。 - **仓库中已有参考证据**：YOLOv8 检测模型在 x86 ONNX 路径完成过模型导入、实时叠加 和事件输出。 - **仍需在目标候选版本上验证**：你的具体模型、Sophon 转换产物、性能、资源占用、 @@ -366,6 +368,29 @@ model_tool --info yolov8n_bm1688_f16.bmodel Sophon 添加模型页面会要求 `.bmodel` 文件，实际页面见下一节的添加模型表单。 +## Rockchip RKNN 路径：共享后端、目标专用产物 + +RK3576 与 RV1126B 共用同一套 CosmoEdge RKNN 推理实现、Rockchip 媒体接口和模型契约； +芯片差异由 `config/rknn/platforms/.json` 平台 profile 提供。模型 spec 不写死芯片， +但每次转换必须绑定一个 profile，因此输出的 `.rknn` 仍是目标专用产物，不能跨芯片复制使用。 + +智能体辅助流程统一使用 `scripts/agent/convert_model.sh` 与 `verify.sh`。执行器按任务合同选择 +RKNN Toolkit2，冻结 Python、wheel、平台 profile、模型 spec、校准集和产物哈希。RK3576 与 +RV1126B 不各自维护一套转换脚本。手工排障时可参考 +[RK3576 RKNN 开发说明](/guide/rk3576-rknn-development)，并把平台参数换成实际目标： + +```bash +python tools/rknn/convert_model.py \ + --spec config/rknn/models/yolov8.json \ + --platform-profile config/rknn/platforms/rv1126b.json \ + --model yolov8-heads.onnx \ + --output yolov8-rv1126b-int8.rknn \ + --quantize --dataset calibration/dataset.txt +``` + +转换主机通过不等于设备验收。必须在对应设备上继续验证 Runtime/驱动、数值输出、图片与视频 +后处理、OSD、规则、告警、5 FPS 目标以及稳定性；这些结果应绑定目标芯片和产物 SHA-256。 + ## 4. 上传并配置模型 ### 4.1 进入模型仓库 diff --git a/scripts/agent/convert_model.sh b/scripts/agent/convert_model.sh index 48dd24c4d..bc04541c9 100755 --- a/scripts/agent/convert_model.sh +++ b/scripts/agent/convert_model.sh @@ -3,4 +3,4 @@ set -euo pipefail PROJECT_ROOT_PATH=$(cd "$(dirname "$0")"; pwd)/../.. -exec python3 "${PROJECT_ROOT_PATH}/tools/model_conversion_workflow.py" convert "$@" +exec python3 "${PROJECT_ROOT_PATH}/tools/conversion_workflow_dispatch.py" convert "$@" diff --git a/scripts/agent/verify.sh b/scripts/agent/verify.sh index 1eafa7175..40369dd58 100755 --- a/scripts/agent/verify.sh +++ b/scripts/agent/verify.sh @@ -3,4 +3,4 @@ set -euo pipefail PROJECT_ROOT_PATH=$(cd "$(dirname "$0")"; pwd)/../.. -exec python3 "${PROJECT_ROOT_PATH}/tools/model_conversion_workflow.py" verify "$@" +exec python3 "${PROJECT_ROOT_PATH}/tools/conversion_workflow_dispatch.py" verify "$@" diff --git a/scripts/build_rknn.sh b/scripts/build_rknn.sh index 845a3aaa0..de3c6464e 100755 --- a/scripts/build_rknn.sh +++ b/scripts/build_rknn.sh @@ -3,20 +3,22 @@ set -euo pipefail export LC_ALL=C.UTF-8 RESOURCE_DIR="" +TARGET_CHIP="${COSMO_TARGET_CHIP:-rk3576}" RKNN_ROOT_PATH="${RKNN_ROOT:-}" ROCKCHIP_MEDIA_ROOT_PATH="${ROCKCHIP_MEDIA_ROOT:-}" RKLLM_ROOT_PATH="${RKLLM_ROOT:-}" RKLLM_REQUIRED="${COSMO_RKLLM_REQUIRED:-OFF}" DEV_MODE=OFF BUILD_TESTS_FLAG=OFF -while getopts "m:r:p:tT" opt; do +while getopts "c:m:r:p:tT" opt; do case ${opt} in + c) TARGET_CHIP="${OPTARG}" ;; m) RESOURCE_DIR="${OPTARG}" ;; r) RKNN_ROOT_PATH="${OPTARG}" ;; p) ROCKCHIP_MEDIA_ROOT_PATH="${OPTARG}" ;; t) DEV_MODE=ON ;; T) BUILD_TESTS_FLAG=ON ;; - *) echo "Usage: $0 -r [-p ] [-m ] [-t] [-T]"; exit 1 ;; + *) echo "Usage: $0 -r [-c rk3576|rv1126b] [-p ] [-m ] [-t] [-T]"; exit 1 ;; esac done @@ -24,12 +26,47 @@ if [ -z "${PROJECT_ROOT_PATH:-}" ]; then PROJECT_ROOT_PATH=$(cd "$(dirname "$0")/.." && pwd) fi +TARGET_CHIP=$(printf '%s' "${TARGET_CHIP}" | tr '[:upper:]' '[:lower:]') +PLATFORM_PROFILE="${PROJECT_ROOT_PATH}/config/rknn/platforms/${TARGET_CHIP}.json" +if [ ! -f "${PLATFORM_PROFILE}" ]; then + echo "ERROR: unsupported RKNN platform profile: ${TARGET_CHIP}" >&2 + exit 1 +fi +IFS=$'\t' read -r PROFILE_CHIP PROFILE_MEDIA_DEFAULT PROFILE_OVERLAY PROFILE_MODELS < <( + python3 - "${PLATFORM_PROFILE}" <<'PY' +import json +import pathlib +import sys + +profile = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +values = ( + profile["chip"], + profile["media"]["default_backend"], + profile["packaging"]["resource_overlay_directory"], + profile["packaging"]["legacy_models_directory"], +) +if any("\t" in str(value) or "\n" in str(value) for value in values): + raise SystemExit("platform profile values must be single-line fields") +print("\t".join(str(value) for value in values)) +PY +) +if [ "${PROFILE_CHIP}" != "${TARGET_CHIP}" ]; then + echo "ERROR: platform profile chip mismatch: ${PROFILE_CHIP} != ${TARGET_CHIP}" >&2 + exit 1 +fi + MEDIA_CPU_BACKEND=ON MEDIA_ROCKCHIP_BACKEND=OFF if [ -n "${ROCKCHIP_MEDIA_ROOT_PATH}" ]; then + python3 "${PROJECT_ROOT_PATH}/tools/rknn/media_sysroot_lock.py" verify \ + --platform-profile "${PLATFORM_PROFILE}" \ + --root "${ROCKCHIP_MEDIA_ROOT_PATH}" MEDIA_CPU_BACKEND=OFF MEDIA_ROCKCHIP_BACKEND=ON fi +if [ "${PROFILE_MEDIA_DEFAULT}" = "rockchip" ] && [ -z "${ROCKCHIP_MEDIA_ROOT_PATH}" ]; then + echo "INFO: ${TARGET_CHIP} profile prefers Rockchip media; using the CPU fallback because no media root was supplied" +fi if [ -z "${RKNN_ROOT_PATH}" ]; then echo "ERROR: pass -r or set RKNN_ROOT" >&2 exit 1 @@ -51,8 +88,20 @@ elif [ "${RESOURCE_DIR#/}" = "${RESOURCE_DIR}" ]; then RESOURCE_DIR="${PROJECT_ROOT_PATH}/${RESOURCE_DIR}" fi -RESOURCE_MODELS_DIR="${PROJECT_ROOT_PATH}/data/resource/aiboxresource_rknn/models" -RESOURCE_OVERLAY_DIR="${PROJECT_ROOT_PATH}/data/resource/aiboxresource_rknn" +RESOURCE_MODELS_DIR="${COSMO_RKNN_MODELS_DIR:-${PROJECT_ROOT_PATH}/${PROFILE_MODELS}}" +RESOURCE_OVERLAY_DIR="${COSMO_RKNN_RESOURCE_OVERLAY_DIR:-${PROJECT_ROOT_PATH}/${PROFILE_OVERLAY}}" +PACKAGE_MODELS="${COSMO_PACKAGE_MODELS:-include}" +if [ "${PACKAGE_MODELS}" = "include" ] && [ ! -d "${RESOURCE_MODELS_DIR}" ]; then + echo "ERROR: target-specific model directory is missing: ${RESOURCE_MODELS_DIR}" >&2 + echo "Convert and stage ${TARGET_CHIP} models first, or set COSMO_PACKAGE_MODELS=preserve for a code-only build." >&2 + exit 1 +fi +if [ "${PACKAGE_MODELS}" = "include" ]; then + python3 "${PROJECT_ROOT_PATH}/tools/rknn/stage_platform_resources.py" \ + --platform-profile "${PLATFORM_PROFILE}" \ + --output-dir "${RESOURCE_OVERLAY_DIR}" \ + --verify +fi BUILD_DIR="${PROJECT_ROOT_PATH}/build_rknn" INSTALL_DIR="${BUILD_DIR}/install" mkdir -p "${BUILD_DIR}" @@ -62,7 +111,7 @@ cmake -S "${PROJECT_ROOT_PATH}" -B "${BUILD_DIR}" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ -DCOSMO_TARGET_ARCH=aarch64 \ - -DCOSMO_TARGET_CHIP=rk3576 \ + -DCOSMO_TARGET_CHIP="${TARGET_CHIP}" \ -DCOSMO_NN_USE_SOPHON_BACKEND=OFF \ -DCOSMO_NN_USE_CPU_BACKEND=OFF \ -DCOSMO_NN_USE_RKNN_BACKEND=ON \ @@ -74,6 +123,7 @@ cmake -S "${PROJECT_ROOT_PATH}" -B "${BUILD_DIR}" \ -DCOSMO_RKLLM_REQUIRED="${RKLLM_REQUIRED}" \ -DCOSMO_ROCKCHIP_MEDIA_ROOT="${ROCKCHIP_MEDIA_ROOT_PATH}" \ -DCOSMO_DEV_MODE="${DEV_MODE}" \ + -DCOSMO_PACKAGE_MODELS="${PACKAGE_MODELS}" \ -DBUILD_TESTS="${BUILD_TESTS_FLAG}" \ -DRESOURCE_DIR="${RESOURCE_DIR}" \ -DRESOURCE_OVERLAY_DIR="${RESOURCE_OVERLAY_DIR}" \ diff --git a/src/api/MessageSystemHandler.cc b/src/api/MessageSystemHandler.cc index 288a7f56d..b545d6c8a 100644 --- a/src/api/MessageSystemHandler.cc +++ b/src/api/MessageSystemHandler.cc @@ -23,6 +23,7 @@ #include "util/FileUtil.h" #include "util/FormatString.h" #include "util/Log.h" +#include "util/NnBackendConstants.h" #include "util/UuidUtil.h" namespace cosmo { @@ -90,6 +91,9 @@ System::MsgQueryDeviceInfoSend MessageSystemHandler::Handle(System::MsgQueryDevi System::MsgQueryDeviceInfoSend retData{}; auto info = device_info_.GetDeviceInfo(); retData.resData.devInfoList.push_back({"deviceType", "设备型号", info.devModel}); + retData.resData.devInfoList.push_back({"acceleratorBackend", "推理后端", cosmo::util::kBackendType}); + retData.resData.devInfoList.push_back( + {"rkllmAvailable", "RKLLM能力", cosmo::util::kSupportsRkllm ? "true" : "false"}); retData.resData.devInfoList.push_back({"hardwareVersion", "固件版本", info.devVersion}); retData.resData.devInfoList.push_back({"softwareVersion", "软件版本", info.softwareVersion}); retData.resData.devInfoList.push_back({"deviceSn", "设备SN", info.devSn}); @@ -155,6 +159,12 @@ System::MsgQueryHardwareResourceSend MessageSystemHandler::Handle( accelerator.mppCopyOutFrames = preview.mpp_copy_out_frames; accelerator.mppCopyOutMs = preview.mpp_copy_out_nanoseconds / 1000000.0; accelerator.mppCopyOutFailures = preview.mpp_copy_out_failures; + accelerator.mppRgaCopyOutFrames = preview.mpp_rga_copy_out_frames; + accelerator.mppRgaCopyOutFailures = preview.mpp_rga_copy_out_failures; + accelerator.mppCpuCopyOutFallbacks = preview.mpp_cpu_copy_out_fallbacks; + accelerator.mppRgaCopyInFrames = preview.mpp_rga_copy_in_frames; + accelerator.mppRgaCopyInFailures = preview.mpp_rga_copy_in_failures; + accelerator.mppCpuCopyInFallbacks = preview.mpp_cpu_copy_in_fallbacks; accelerator.mppEarlyDroppedFrames = preview.mpp_early_dropped_frames; const auto inference = nn::GetInferencePipelineMetrics().Snapshot(); accelerator.colorConvertFrames = inference.color_convert_frames; @@ -201,16 +211,23 @@ System::MsgQueryHardwareResourceSend MessageSystemHandler::Handle( accelerator.rknnDetectorOutputTransformCalls = inference.rknn_detector_output_transform_calls; accelerator.rknnDetectorOutputTransformMs = inference.rknn_detector_output_transform_nanoseconds / 1000000.0; - accelerator.rknnDetectorMutexWaitCalls = inference.rknn_detector_mutex_wait_calls; - accelerator.rknnDetectorMutexWaitMs = inference.rknn_detector_mutex_wait_nanoseconds / 1000000.0; - accelerator.rknnPreprocessFastHits = inference.rknn_preprocess_fast_hits; - accelerator.rknnRgaFillCalls = inference.rknn_rga_fill_calls; - accelerator.rknnRgaFillMs = inference.rknn_rga_fill_nanoseconds / 1000000.0; - accelerator.rknnRgaResizeColorCalls = inference.rknn_rga_resize_color_calls; - accelerator.rknnRgaResizeColorMs = inference.rknn_rga_resize_color_nanoseconds / 1000000.0; - accelerator.rknnRgaFailures = inference.rknn_rga_failures; - accelerator.rknnCpuResizeFallbackCalls = inference.rknn_cpu_resize_fallback_calls; - accelerator.rknnCpuResizeFallbackMs = inference.rknn_cpu_resize_fallback_nanoseconds / 1000000.0; + accelerator.rknnDetectorMutexWaitCalls = inference.rknn_detector_mutex_wait_calls; + accelerator.rknnDetectorMutexWaitMs = inference.rknn_detector_mutex_wait_nanoseconds / 1000000.0; + accelerator.rknnPreprocessFastHits = inference.rknn_preprocess_fast_hits; + accelerator.rknnRgaFillCalls = inference.rknn_rga_fill_calls; + accelerator.rknnRgaFillMs = inference.rknn_rga_fill_nanoseconds / 1000000.0; + accelerator.rknnRgaResizeColorCalls = inference.rknn_rga_resize_color_calls; + accelerator.rknnRgaResizeColorMs = inference.rknn_rga_resize_color_nanoseconds / 1000000.0; + accelerator.rknnRgaCropResizeCalls = inference.rknn_rga_crop_resize_calls; + accelerator.rknnRgaCropResizeMs = inference.rknn_rga_crop_resize_nanoseconds / 1000000.0; + accelerator.rknnRgaCropResizeFailures = inference.rknn_rga_crop_resize_failures; + accelerator.rknnRgaCropDmaBufFrames = inference.rknn_rga_crop_dmabuf_frames; + accelerator.rknnRgaCropHostFallbacks = inference.rknn_rga_crop_host_fallbacks; + accelerator.rknnRgaFailures = inference.rknn_rga_failures; + accelerator.rknnCpuResizeFallbackCalls = inference.rknn_cpu_resize_fallback_calls; + accelerator.rknnCpuResizeFallbackMs = inference.rknn_cpu_resize_fallback_nanoseconds / 1000000.0; + accelerator.rknnCpuCropResizeFallbackCalls = inference.rknn_cpu_crop_resize_fallback_calls; + accelerator.rknnCpuCropResizeFallbackMs = inference.rknn_cpu_crop_resize_fallback_nanoseconds / 1000000.0; accelerator.rknnCpuNormalizeFallbackCalls = inference.rknn_cpu_normalize_fallback_calls; accelerator.rknnCpuNormalizeFallbackMs = inference.rknn_cpu_normalize_fallback_nanoseconds / 1000000.0; accelerator.rknnNativeInputMapCalls = inference.rknn_native_input_map_calls; diff --git a/src/flow/classify/AiClassifier.cc b/src/flow/classify/AiClassifier.cc index ef87aa7e9..4b86c3fef 100644 --- a/src/flow/classify/AiClassifier.cc +++ b/src/flow/classify/AiClassifier.cc @@ -67,6 +67,7 @@ bool AiClassifier::CheckDataAvailable(AlgDataPtr algData) { void AiClassifier::HandFramesEx(std::vector alg_datas) { std::vector alg_out_datas; std::vector in_images; + std::vector in_native_buffers; std::vector> io_puts; std::vector img_record; size_t image_num = alg_datas.size(); @@ -146,10 +147,12 @@ void AiClassifier::HandFramesEx(std::vector alg_datas) { io_puts.push_back(io_puts_el); in_images.push_back(alg_data->chanDataDec.frame); + in_native_buffers.push_back(alg_data->chanDataDec.native_buffer); img_record.push_back(i); } - action_status = classifier_->ClassifyMultSub(in_images, io_puts, target_have_mult_related, true); + action_status = + classifier_->ClassifyMultSub(in_images, io_puts, target_have_mult_related, true, in_native_buffers); for (size_t out = 0; out < io_puts.size(); out++) { auto classify_result = alg_out_datas[img_record[out]]->GetTaskResult(AlgDataType::TaskDataClassify); if (classify_result) { diff --git a/src/flow/common/AlgDataQueueDistributor.cc b/src/flow/common/AlgDataQueueDistributor.cc index 12242c5f6..992cd0b1c 100644 --- a/src/flow/common/AlgDataQueueDistributor.cc +++ b/src/flow/common/AlgDataQueueDistributor.cc @@ -300,12 +300,9 @@ int AlgDataQueueDistributor::DistributorPreparedFrame( int message_count = 0; for (const auto& queue : plan.queues) { - // Native inference buffers are released by each detector immediately - // after its synchronous Forward. Give parallel detector queues their - // own AlgData wrapper so one consumer cannot clear another consumer's - // borrowed DMA-BUF descriptor. VideoFrame and the native owner remain - // shared; the ordinary single-queue path keeps its existing allocation - // behavior. + // Give parallel detector queues their own AlgData wrapper. The native + // owner is shared and remains valid through downstream classifier + // preprocessing, while per-branch task state stays isolated. auto queued = converted; if (plan.queues.size() > 1 && converted->chanDataDec.native_buffer) { queued = AlgDataCopy(converted); diff --git a/src/flow/detect/AiDetector.cc b/src/flow/detect/AiDetector.cc index 68ed58421..2b50c5ea6 100644 --- a/src/flow/detect/AiDetector.cc +++ b/src/flow/detect/AiDetector.cc @@ -309,9 +309,6 @@ void AiDetector::HandFrames(std::vector alg_datas) { } action_status = detector_->Detect(images, native_buffers, confThres, detRsts); native_buffers.clear(); - for (auto& alg_data : activeAlgDatas) { - alg_data->chanDataDec.native_buffer.reset(); - } if (util::ErrorEnum::Success != action_status) { LOG_ERRO("{}[{} {}] Detect Failed. Ret:{} images:{} confThres:{}", kTag, name_, uuid, action_status, images.size(), confThres.size()); diff --git a/src/flow/detect/AiDetectorFps.h b/src/flow/detect/AiDetectorFps.h index 8c505a0a5..41816a7aa 100644 --- a/src/flow/detect/AiDetectorFps.h +++ b/src/flow/detect/AiDetectorFps.h @@ -52,9 +52,9 @@ namespace ai_detector_fps { return {{8.0f, 3}, {12.0f, 2}, {100.0f, 1}}; } - // RKNN CV graphs on RK3576 are admitted conservatively until a - // model/candidate-specific capacity result proves that sharing is safe. - // Environment overrides remain available for qualified models. + // RKNN CV graphs use a conservative default across Rockchip targets until + // a model/candidate-specific capacity result proves that sharing is safe. + // Environment overrides remain available for qualified combinations. inline ReuseProfile RknnDefaultReuseProfile() { return {{100.0f, 1}}; } diff --git a/src/infer/AiClassifierForward.cc b/src/infer/AiClassifierForward.cc index 1ec2df8bb..5eac1b02d 100644 --- a/src/infer/AiClassifierForward.cc +++ b/src/infer/AiClassifierForward.cc @@ -128,9 +128,10 @@ util::ErrorEnum AiClassifierUnify::Classify(const std::vector& im util::ErrorEnum AiClassifierUnify::Forward(const std::vector& images, const std::vector>>& datas, std::vector>& outputs, - bool use_box) { + bool use_box, + const std::vector& native_buffers) { std::vector> image_blobs{}; - auto ret = ConvertImagesToBlobs(images, image_blobs); + auto ret = ConvertImagesToBlobs(images, native_buffers, image_blobs); if (util::ErrorEnum::Success != ret) { LOG_ERRO("ConvertImagesToBlobs Failed. Ret:{}", ret); return ret; diff --git a/src/infer/AiClassifierUnify.cc b/src/infer/AiClassifierUnify.cc index 3e2ddf660..2fe08284b 100644 --- a/src/infer/AiClassifierUnify.cc +++ b/src/infer/AiClassifierUnify.cc @@ -95,7 +95,8 @@ void AiClassifierUnify::AppendClassifyResults(AiDetectRstEl& io_el, /* Input data must not exceed hardware-supported max batch; most edge devices support single batch only */ util::ErrorEnum AiClassifierUnify::Classify(const std::vector& images, - std::vector& io_rst, bool use_box) { + std::vector& io_rst, bool use_box, + const std::vector& native_buffers) { if (!classifier_) { LOG_WARN("{}", "SDK Classifier Not Init"); return util::ErrorEnum::NotInit; @@ -109,7 +110,7 @@ util::ErrorEnum AiClassifierUnify::Classify(const std::vector& im } std::vector> image_blobs{}; - auto ret = ConvertImagesToBlobs(images, image_blobs); + auto ret = ConvertImagesToBlobs(images, native_buffers, image_blobs); if (util::ErrorEnum::Success != ret) { LOG_ERRO("ConvertImagesToBlobs Failed. Ret:{}", ret); return ret; @@ -146,11 +147,11 @@ util::ErrorEnum AiClassifierUnify::Classify(const std::vector& im return util::ErrorEnum::Success; } -util::ErrorEnum AiClassifierUnify::ClassifyMultSub(const std::vector& images, - std::vector>& io_rst, - const bool& use_mult_sub, bool use_box) { +util::ErrorEnum AiClassifierUnify::ClassifyMultSub( + const std::vector& images, std::vector>& io_rst, + const bool& use_mult_sub, bool use_box, const std::vector& native_buffers) { if (!use_mult_sub) { - return Classify(images, io_rst, use_box); + return Classify(images, io_rst, use_box, native_buffers); } if (io_rst.empty()) { return util::ErrorEnum::Success; @@ -163,6 +164,8 @@ util::ErrorEnum AiClassifierUnify::ClassifyMultSub(const std::vector imageInputs; imageInputs.push_back(images[i]); + std::vector nativeInputs; + nativeInputs.push_back(i < native_buffers.size() ? native_buffers[i] : nullptr); for (size_t j = 0; j < io_rst[i].size(); j++) { for (size_t k = 0; k < io_rst[i][j].relatedEls.size(); k++) { AiDetectRstEl el; @@ -170,7 +173,7 @@ util::ErrorEnum AiClassifierUnify::ClassifyMultSub(const std::vector io_puts; io_puts.push_back(el); - auto ret = Classify(imageInputs, io_puts, use_box); + auto ret = Classify(imageInputs, io_puts, use_box, nativeInputs); if (util::ErrorEnum::Success != ret) { LOG_INFO("{}", "Classify Fail"); return ret; @@ -213,7 +216,8 @@ void AiClassifierUnify::DispatchBatchResults(const std::vector& images, - std::vector>& io_rst, bool use_box) { + std::vector>& io_rst, bool use_box, + const std::vector& native_buffers) { if (!classifier_) { LOG_WARN("{}", "SDK Classifier Not Init"); return util::ErrorEnum::NotInit; @@ -227,6 +231,7 @@ util::ErrorEnum AiClassifierUnify::Classify(const std::vector& im } std::vector input_images; + std::vector input_native_buffers; std::vector>> input_datas; std::vector> indexes; size_t total = std::accumulate(io_rst.begin(), io_rst.end(), size_t{0}, @@ -245,15 +250,18 @@ util::ErrorEnum AiClassifierUnify::Classify(const std::vector& im if (img != indexes[k].first) { img = indexes[k].first; input_images.push_back(images[img]); + input_native_buffers.push_back(img < native_buffers.size() ? native_buffers[img] + : nullptr); } } std::vector> outputs; - auto ret = Forward(input_images, input_datas, outputs, use_box); + auto ret = Forward(input_images, input_datas, outputs, use_box, input_native_buffers); if (util::ErrorEnum::Success != ret) { LOG_ERRO("Forward Failed. Ret:{}", ret); } DispatchBatchResults(outputs, indexes, io_rst); input_images.clear(); + input_native_buffers.clear(); input_datas.clear(); indexes.clear(); datas.clear(); diff --git a/src/infer/AiClassifierUnify.h b/src/infer/AiClassifierUnify.h index 18debf5f8..dab28ff77 100644 --- a/src/infer/AiClassifierUnify.h +++ b/src/infer/AiClassifierUnify.h @@ -25,14 +25,17 @@ class AiClassifierUnify { bool use_box = true); util::ErrorEnum Classify(const std::vector& images, std::vector& io_rst, - bool use_box = true); + bool use_box = true, + const std::vector& native_buffers = {}); util::ErrorEnum ClassifyMultSub(const std::vector& images, std::vector>& io_rst, const bool& use_mult_sub, - bool use_box = true); + bool use_box = true, + const std::vector& native_buffers = {}); util::ErrorEnum Classify(const std::vector& images, - std::vector>& io_rst, bool use_box = true); + std::vector>& io_rst, bool use_box = true, + const std::vector& native_buffers = {}); util::ErrorEnum Classify(const VideoFramePtr& image_base, const VideoFramePtr& image, AiDetectRstEl& io_rst); @@ -49,7 +52,8 @@ class AiClassifierUnify { private: util::ErrorEnum Forward(const std::vector& images, const std::vector>>& datas, - std::vector>& outputs, bool use_box); + std::vector>& outputs, bool use_box, + const std::vector& native_buffers); std::vector CollectBoxOrLandmarkData(const AiDetectRstEl& io_el, bool use_box); void AppendClassifyResults(AiDetectRstEl& io_el, const std::vector& obj); diff --git a/src/infer/AiComponment.cc b/src/infer/AiComponment.cc index 8ee744ffa..6789322fc 100644 --- a/src/infer/AiComponment.cc +++ b/src/infer/AiComponment.cc @@ -125,6 +125,18 @@ util::ErrorEnum ConvertImagesToBlobs(const std::vector& images, handle.native_image.height = native.height; handle.native_image.width_stride = native.width_stride; handle.native_image.height_stride = native.height_stride; + if (native.color_space == media::NativeVideoColorSpace::Bt601) { + handle.native_image.color_space = cosmo::nn::NativeImageColorSpace::Bt601; + } else if (native.color_space == media::NativeVideoColorSpace::Bt709) { + handle.native_image.color_space = cosmo::nn::NativeImageColorSpace::Bt709; + } else if (native.color_space == media::NativeVideoColorSpace::Bt2020) { + handle.native_image.color_space = cosmo::nn::NativeImageColorSpace::Bt2020; + } + if (native.color_range == media::NativeVideoColorRange::Limited) { + handle.native_image.color_range = cosmo::nn::NativeImageColorRange::Limited; + } else if (native.color_range == media::NativeVideoColorRange::Full) { + handle.native_image.color_range = cosmo::nn::NativeImageColorRange::Full; + } if (native.format == media::NativeVideoBufferFormat::NV12) { handle.native_image.format = cosmo::nn::IMAGE_NV12; } else if (native.format == media::NativeVideoBufferFormat::I420) { diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt index 2d3a8bc44..f8a4ac8be 100644 --- a/src/media/CMakeLists.txt +++ b/src/media/CMakeLists.txt @@ -47,10 +47,12 @@ set(MEDIA_CPU_SRC VideoFrameProcCpu.cc VideoFrameProcCpu.h ) -# Rockchip Copy-first backend. MPP handles H.264/H.265 decode and H.264 encode, -# RGA handles admitted color-conversion/resize operations, and FFmpeg remains a -# deterministic fallback. No DMA-BUF zero-copy path is compiled in this phase. +# Rockchip hardware backend. MPP handles H.264/H.265 decode and H.264 encode, +# RGA handles admitted color-conversion/resize operations, and the RKNN fast +# path can bind MPP/RGA DMA-BUF inputs directly. FFmpeg and CPU transforms +# remain explicit fault-tolerance fallbacks. set(MEDIA_ROCKCHIP_SRC + RockchipRgaBuffer.h VideoDecoderCpu.cc VideoDecoderCpu.h VideoDecoderCreateRockchip.cc VideoDecoderRockchip.cc VideoDecoderRockchip.h diff --git a/src/media/NativeVideoBuffer.h b/src/media/NativeVideoBuffer.h index 2db070a97..bc71903ad 100644 --- a/src/media/NativeVideoBuffer.h +++ b/src/media/NativeVideoBuffer.h @@ -12,6 +12,19 @@ enum class NativeVideoBufferFormat { NV21, }; +enum class NativeVideoColorSpace { + Unspecified = 0, + Bt601, + Bt709, + Bt2020, +}; + +enum class NativeVideoColorRange { + Unspecified = 0, + Limited, + Full, +}; + /// Optional, backend-owned image buffer presented to a hardware consumer. /// /// The file descriptor is borrowed from owner. Keeping this object alive keeps @@ -24,6 +37,8 @@ struct NativeVideoBuffer { int width_stride{0}; int height_stride{0}; NativeVideoBufferFormat format{NativeVideoBufferFormat::Unknown}; + NativeVideoColorSpace color_space{NativeVideoColorSpace::Unspecified}; + NativeVideoColorRange color_range{NativeVideoColorRange::Unspecified}; std::shared_ptr owner; [[nodiscard]] bool Valid() const { diff --git a/src/media/PreviewPipelineMetrics.cc b/src/media/PreviewPipelineMetrics.cc index 854bbec22..a11a71244 100644 --- a/src/media/PreviewPipelineMetrics.cc +++ b/src/media/PreviewPipelineMetrics.cc @@ -97,6 +97,24 @@ void PreviewPipelineMetrics::RecordMppCopyOut(bool success, uint64_t nanoseconds } } +void PreviewPipelineMetrics::RecordMppRgaCopyOut(bool success) { + auto& counter = success ? mpp_rga_copy_out_frames_ : mpp_rga_copy_out_failures_; + counter.fetch_add(1, std::memory_order_relaxed); +} + +void PreviewPipelineMetrics::RecordMppCpuCopyOutFallback() { + mpp_cpu_copy_out_fallbacks_.fetch_add(1, std::memory_order_relaxed); +} + +void PreviewPipelineMetrics::RecordMppRgaCopyIn(bool success) { + auto& counter = success ? mpp_rga_copy_in_frames_ : mpp_rga_copy_in_failures_; + counter.fetch_add(1, std::memory_order_relaxed); +} + +void PreviewPipelineMetrics::RecordMppCpuCopyInFallback() { + mpp_cpu_copy_in_fallbacks_.fetch_add(1, std::memory_order_relaxed); +} + void PreviewPipelineMetrics::RecordMppEarlyDrop() { mpp_early_dropped_frames_.fetch_add(1, std::memory_order_relaxed); } @@ -130,6 +148,12 @@ PreviewPipelineMetricsSnapshot PreviewPipelineMetrics::Snapshot() const { mpp_copy_out_frames_.load(std::memory_order_relaxed), mpp_copy_out_nanoseconds_.load(std::memory_order_relaxed), mpp_copy_out_failures_.load(std::memory_order_relaxed), + mpp_rga_copy_out_frames_.load(std::memory_order_relaxed), + mpp_rga_copy_out_failures_.load(std::memory_order_relaxed), + mpp_cpu_copy_out_fallbacks_.load(std::memory_order_relaxed), + mpp_rga_copy_in_frames_.load(std::memory_order_relaxed), + mpp_rga_copy_in_failures_.load(std::memory_order_relaxed), + mpp_cpu_copy_in_fallbacks_.load(std::memory_order_relaxed), mpp_early_dropped_frames_.load(std::memory_order_relaxed), }; } diff --git a/src/media/PreviewPipelineMetrics.h b/src/media/PreviewPipelineMetrics.h index b9f3d2d4f..3ea8b3a3b 100644 --- a/src/media/PreviewPipelineMetrics.h +++ b/src/media/PreviewPipelineMetrics.h @@ -33,6 +33,12 @@ struct PreviewPipelineMetricsSnapshot { uint64_t mpp_copy_out_frames{0}; uint64_t mpp_copy_out_nanoseconds{0}; uint64_t mpp_copy_out_failures{0}; + uint64_t mpp_rga_copy_out_frames{0}; + uint64_t mpp_rga_copy_out_failures{0}; + uint64_t mpp_cpu_copy_out_fallbacks{0}; + uint64_t mpp_rga_copy_in_frames{0}; + uint64_t mpp_rga_copy_in_failures{0}; + uint64_t mpp_cpu_copy_in_fallbacks{0}; uint64_t mpp_early_dropped_frames{0}; }; @@ -50,6 +56,10 @@ class PreviewPipelineMetrics { void RecordMppDecode(bool success, uint64_t nanoseconds); void RecordMppDecodeFallback(); void RecordMppCopyOut(bool success, uint64_t nanoseconds); + void RecordMppRgaCopyOut(bool success); + void RecordMppCpuCopyOutFallback(); + void RecordMppRgaCopyIn(bool success); + void RecordMppCpuCopyInFallback(); void RecordMppEarlyDrop(); [[nodiscard]] PreviewPipelineMetricsSnapshot Snapshot() const; @@ -82,6 +92,12 @@ class PreviewPipelineMetrics { std::atomic mpp_copy_out_frames_{0}; std::atomic mpp_copy_out_nanoseconds_{0}; std::atomic mpp_copy_out_failures_{0}; + std::atomic mpp_rga_copy_out_frames_{0}; + std::atomic mpp_rga_copy_out_failures_{0}; + std::atomic mpp_cpu_copy_out_fallbacks_{0}; + std::atomic mpp_rga_copy_in_frames_{0}; + std::atomic mpp_rga_copy_in_failures_{0}; + std::atomic mpp_cpu_copy_in_fallbacks_{0}; std::atomic mpp_early_dropped_frames_{0}; }; diff --git a/src/media/RockchipRgaBuffer.h b/src/media/RockchipRgaBuffer.h new file mode 100644 index 000000000..327348d7c --- /dev/null +++ b/src/media/RockchipRgaBuffer.h @@ -0,0 +1,113 @@ +#pragma once + +#include +#include + +#include +#include + +namespace cosmo::media { + +/// One imported RGA buffer handle with deterministic release semantics. +/// +/// The wrapper is intentionally backend-wide rather than SoC-specific. It is +/// used for MPP DMA-BUFs, RKNN DMA-BUFs, and the existing host-frame boundary. +class ScopedRgaBufferHandle { +public: + ScopedRgaBufferHandle() = default; + + ScopedRgaBufferHandle(void* address, size_t bytes) { + ImportVirtual(address, bytes); + } + + ~ScopedRgaBufferHandle() { + Reset(); + } + + ScopedRgaBufferHandle(const ScopedRgaBufferHandle&) = delete; + ScopedRgaBufferHandle& operator=(const ScopedRgaBufferHandle&) = delete; + + ScopedRgaBufferHandle(ScopedRgaBufferHandle&& other) noexcept : handle_(other.handle_) { + other.handle_ = 0; + } + + ScopedRgaBufferHandle& operator=(ScopedRgaBufferHandle&& other) noexcept { + if (this != &other) { + Reset(); + handle_ = other.handle_; + other.handle_ = 0; + } + return *this; + } + + bool ImportVirtual(void* address, size_t bytes) { + if (handle_ != 0 || !address || !ValidSize(bytes)) { + return false; + } + handle_ = importbuffer_virtualaddr(address, static_cast(bytes)); + return handle_ != 0; + } + + bool ImportFd(int fd, size_t bytes) { + if (handle_ != 0 || fd < 0 || !ValidSize(bytes)) { + return false; + } + handle_ = importbuffer_fd(fd, static_cast(bytes)); + return handle_ != 0; + } + + void Reset() { + if (handle_ != 0) { + releasebuffer_handle(handle_); + handle_ = 0; + } + } + + [[nodiscard]] rga_buffer_handle_t Get() const { + return handle_; + } + + [[nodiscard]] explicit operator bool() const { + return handle_ != 0; + } + +private: + static bool ValidSize(size_t bytes) { + return bytes > 0 && bytes <= static_cast(std::numeric_limits::max()); + } + + rga_buffer_handle_t handle_{0}; +}; + +inline bool RockchipRgaSucceeded(IM_STATUS status) { + return status == IM_STATUS_SUCCESS || status == IM_STATUS_NOERROR; +} + +inline constexpr bool RockchipRgaHasBt2020ColorSpace() { +#if defined(RGA_CURRENT_API_VERSION) && RGA_CURRENT_API_VERSION >= 0x010a0600 + return true; +#else + return false; +#endif +} + +inline IM_COLOR_SPACE_MODE RockchipRgaBt2020ColorSpace(bool full_range) { +#if defined(RGA_CURRENT_API_VERSION) && RGA_CURRENT_API_VERSION >= 0x010a0600 + return full_range ? IM_YUV_BT2020_FULL_RANGE : IM_YUV_BT2020_LIMIT_RANGE; +#else + // librga 1.10.1 and earlier do not expose BT.2020 full-CSC modes. Keep + // compilation and runtime behavior inside the advertised header contract; + // the caller emits a warning if this colorimetry downgrade is exercised. + return full_range ? IM_YUV_BT709_FULL_RANGE : IM_YUV_BT709_LIMIT_RANGE; +#endif +} + +inline void SetRgaYuvToRgbColorSpace(rga_buffer_t& source, rga_buffer_t& target, + IM_COLOR_SPACE_MODE source_mode = IM_YUV_BT601_LIMIT_RANGE) { + imsetColorSpace(&source, source_mode); + // IM_RGB_FULL_RANGE is a newer alias; IM_RGB_FULL is ABI-identical and is + // present in both supported librga header generations. + imsetColorSpace(&target, IM_RGB_FULL); +} + +} // namespace cosmo::media diff --git a/src/media/VideoDecoderRockchip.cc b/src/media/VideoDecoderRockchip.cc index 1cd82de02..a690fb67b 100644 --- a/src/media/VideoDecoderRockchip.cc +++ b/src/media/VideoDecoderRockchip.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include "media/PreviewPipelineMetrics.h" +#include "media/RockchipRgaBuffer.h" #include "media/VideoDecoderCpu.h" #include "util/Log.h" @@ -66,6 +68,46 @@ namespace { return base == MPP_FMT_YUV420P || base == MPP_FMT_YUV420SP || base == MPP_FMT_YUV420SP_VU; } + int ToRga420Format(MppFrameFormat format) { + const auto base = static_cast(format) & MPP_FRAME_FMT_MASK; + if (base == MPP_FMT_YUV420SP) { + return RK_FORMAT_YCbCr_420_SP; + } + if (base == MPP_FMT_YUV420SP_VU) { + return RK_FORMAT_YCrCb_420_SP; + } + if (base == MPP_FMT_YUV420P) { + return RK_FORMAT_YCbCr_420_P; + } + return RK_FORMAT_UNKNOWN; + } + + NativeVideoColorSpace ToNativeColorSpace(MppFrameColorSpace color_space) { + switch (color_space) { + case MPP_FRAME_SPC_BT709: + return NativeVideoColorSpace::Bt709; + case MPP_FRAME_SPC_BT470BG: + case MPP_FRAME_SPC_SMPTE170M: + case MPP_FRAME_SPC_SMPTE240M: + return NativeVideoColorSpace::Bt601; + case MPP_FRAME_SPC_BT2020_NCL: + case MPP_FRAME_SPC_BT2020_CL: + return NativeVideoColorSpace::Bt2020; + default: + return NativeVideoColorSpace::Unspecified; + } + } + + NativeVideoColorRange ToNativeColorRange(MppFrameColorRange color_range) { + if (color_range == MPP_FRAME_RANGE_MPEG) { + return NativeVideoColorRange::Limited; + } + if (color_range == MPP_FRAME_RANGE_JPEG) { + return NativeVideoColorRange::Full; + } + return NativeVideoColorRange::Unspecified; + } + NativeVideoBufferPtr ExportMppBuffer(const std::string& decoder_name, MppFrame frame) { if (!frame) { return nullptr; @@ -109,6 +151,8 @@ namespace { result->width_stride = static_cast(horizontal_stride); result->height_stride = static_cast(vertical_stride); result->format = native_format; + result->color_space = ToNativeColorSpace(mpp_frame_get_colorspace(frame)); + result->color_range = ToNativeColorRange(mpp_frame_get_color_range(frame)); result->owner = std::shared_ptr(buffer, [](void* value) { if (value) { mpp_buffer_put(static_cast(value)); @@ -117,7 +161,7 @@ namespace { return result; } - VideoFramePtr CopyMppFrame(const std::string& decoder_name, MppFrame frame) { + VideoFramePtr CopyMppFrameCpu(const std::string& decoder_name, MppFrame frame) { if (!frame) { return nullptr; } @@ -204,6 +248,78 @@ namespace { return output; } + VideoFramePtr MaterializeMppFrame(const std::string& decoder_name, MppFrame frame) { + if (!frame) { + return nullptr; + } + + const size_t width = mpp_frame_get_width(frame); + const size_t height = mpp_frame_get_height(frame); + const size_t horizontal_stride = mpp_frame_get_hor_stride(frame); + const size_t vertical_stride = mpp_frame_get_ver_stride(frame); + const auto format = mpp_frame_get_fmt(frame); + auto buffer = mpp_frame_get_buffer(frame); + const int source_format = ToRga420Format(format); + if (width == 0 || height == 0 || width % 2 != 0 || height % 2 != 0 || + width > static_cast(std::numeric_limits::max()) || + height > static_cast(std::numeric_limits::max()) || + horizontal_stride > static_cast(std::numeric_limits::max()) || + vertical_stride > static_cast(std::numeric_limits::max()) || + horizontal_stride < width || vertical_stride < height || !IsCompact420Format(format) || !buffer || + source_format == RK_FORMAT_UNKNOWN) { + GetPreviewPipelineMetrics().RecordMppRgaCopyOut(false); + GetPreviewPipelineMetrics().RecordMppCpuCopyOutFallback(); + return CopyMppFrameCpu(decoder_name, frame); + } + + auto output = std::make_shared(static_cast(width), static_cast(height), + PixelFormat::PIXEL_I420); + if (!output || !output->Active() || !output->GetData()) { + GetPreviewPipelineMetrics().RecordMppRgaCopyOut(false); + return nullptr; + } + + const auto rga_started = std::chrono::steady_clock::now(); + ScopedRgaBufferHandle source_handle; + ScopedRgaBufferHandle target_handle(output->GetData(), output->GetSize()); + const int source_fd = mpp_buffer_get_fd(buffer); + source_handle.ImportFd(source_fd, mpp_buffer_get_size(buffer)); + IM_STATUS status = IM_STATUS_OUT_OF_MEMORY; + if (source_handle && target_handle) { + auto source = wrapbuffer_handle_t(source_handle.Get(), static_cast(width), + static_cast(height), static_cast(horizontal_stride), + static_cast(vertical_stride), source_format); + auto target = + wrapbuffer_handle_t(target_handle.Get(), static_cast(width), static_cast(height), + static_cast(width), static_cast(height), RK_FORMAT_YCbCr_420_P); + if (source_format == RK_FORMAT_YCbCr_420_P) { + const im_rect source_rect{0, 0, static_cast(width), static_cast(height)}; + const im_rect target_rect = source_rect; + const im_rect empty_rect{}; + const rga_buffer_t empty_buffer{}; + status = + improcess(source, target, empty_buffer, source_rect, target_rect, empty_rect, IM_SYNC); + } else { + status = imcvtcolor_t(source, target, source_format, RK_FORMAT_YCbCr_420_P, + IM_COLOR_SPACE_DEFAULT, 1); + } + } + const bool rga_success = RockchipRgaSucceeded(status); + GetPreviewPipelineMetrics().RecordRgaOperation(rga_success, ElapsedNanoseconds(rga_started)); + GetPreviewPipelineMetrics().RecordMppRgaCopyOut(rga_success); + if (rga_success) { + return output; + } + + static std::atomic_flag fallback_logged = ATOMIC_FLAG_INIT; + if (!fallback_logged.test_and_set(std::memory_order_relaxed)) { + LOG_WARN("{} RGA DMA-BUF materialization failed with status {} ({}); using CPU copy-out", + decoder_name, status, imStrError_t(status)); + } + GetPreviewPipelineMetrics().RecordMppCpuCopyOutFallback(); + return CopyMppFrameCpu(decoder_name, frame); + } + } // namespace struct PendingDecodeTiming { @@ -231,7 +347,7 @@ VideoDecoderRockchip::~VideoDecoderRockchip() { VideoDecoderCapability VideoDecoderRockchip::Probe(VideoCodecType type) { VideoDecoderCapability capability; - capability.backend = "rockchip-copy-out"; + capability.backend = "rockchip-mpp-rga"; capability.implementation = "rockchip-mpp-vpu"; const auto coding = ToMppCoding(type); @@ -250,7 +366,9 @@ VideoDecoderCapability VideoDecoderRockchip::Probe(VideoCodecType type) { const bool format_supported = mpp_check_support_format(MPP_CTX_DEC, coding) == MPP_OK; if (device_accessible && format_supported) { capability.available = true; - capability.detail = "MPP VPU decoder is available; decoded frames are copied out as compact I420"; + capability.detail = + "MPP VPU decode and DMA-BUF export are available; selected host frames are materialized " + "through RGA with an observable CPU fallback"; return capability; } @@ -346,7 +464,7 @@ bool VideoDecoderRockchip::OpenMpp() { return false; } - // NV12 is the native RK3576 decoder output. It is deinterleaved during the + // NV12 is the native Rockchip decoder output. It is deinterleaved during the // explicit Copy-out boundary, so downstream code still receives I420. MppFrameFormat output_format = MPP_FMT_YUV420SP; ret = state_->api->control(state_->context, MPP_DEC_SET_OUTPUT_FORMAT, &output_format); @@ -365,7 +483,8 @@ bool VideoDecoderRockchip::OpenMpp() { } state_->opened = true; - LOG_INFO("{} MPP VPU decoder opened: codec={} copy-out=I420", idx_name_, static_cast(codec_type_)); + LOG_INFO("{} MPP VPU decoder opened: codec={} materialize=RGA-I420", idx_name_, + static_cast(codec_type_)); return true; } @@ -589,7 +708,7 @@ DecodedVideoFrame VideoDecoderRockchip::ReceiveMppFrame(bool& made_progress) { PixelFormat::PIXEL_I420, [holder, decoder_name, resolved_pts]() { const auto copy_started = std::chrono::steady_clock::now(); - auto output = CopyMppFrame(decoder_name, holder->frame); + auto output = MaterializeMppFrame(decoder_name, holder->frame); if (output) { output->SetFrameIndex(static_cast(std::max(0, resolved_pts))); } diff --git a/src/media/VideoDecoderRockchip.h b/src/media/VideoDecoderRockchip.h index 6e7aa8abe..6860b48ba 100644 --- a/src/media/VideoDecoderRockchip.h +++ b/src/media/VideoDecoderRockchip.h @@ -9,13 +9,13 @@ namespace cosmo::media { class VideoDecoderCpu; struct RockchipDecoderState; -/// Rockchip MPP decoder with a bounded device-to-host copy boundary. +/// Shared Rockchip MPP decoder with native DMA-BUF export and RGA materialization. /// /// Compressed H.264/H.265 packets are decoded by the VPU. A decoded MPP frame /// is retained on the decoder thread until the channel sampling decision; only -/// selected outputs are copied into the compact I420 VideoFrame expected by -/// the existing CosmoEdge pipeline. DMA-BUF zero-copy ownership is -/// intentionally outside this phase. +/// selected outputs are materialized into the compact I420 VideoFrame expected +/// by host-only business consumers. RKNN preprocessing consumes the retained +/// DMA-BUF directly; RGA performs the unavoidable host-bound materialization. class VideoDecoderRockchip final : public VideoDecoder { public: explicit VideoDecoderRockchip(size_t name); diff --git a/src/media/VideoEncoderRockchip.cc b/src/media/VideoEncoderRockchip.cc index d9ac9d9b5..c719f194a 100644 --- a/src/media/VideoEncoderRockchip.cc +++ b/src/media/VideoEncoderRockchip.cc @@ -19,6 +19,7 @@ #include #include "media/PreviewPipelineMetrics.h" +#include "media/RockchipRgaBuffer.h" #include "media/VideoEncoderCpu.h" #include "util/Log.h" @@ -100,6 +101,8 @@ struct RockchipEncoderState { size_t vertical_stride{0}; size_t frame_buffer_size{0}; int64_t frame_pts{0}; + ScopedRgaBufferHandle frame_rga_handle; + bool rga_copy_in_available{false}; std::vector codec_header; }; @@ -111,7 +114,7 @@ VideoEncoderRockchip::~VideoEncoderRockchip() { VideoEncoderCapability VideoEncoderRockchip::Probe(VideoCodecType type) { VideoEncoderCapability capability; - capability.backend = "rockchip-copy-first"; + capability.backend = "rockchip-mpp-rga"; capability.implementation = "rockchip-mpp"; const auto coding = ToMppCoding(type); @@ -131,8 +134,8 @@ VideoEncoderCapability VideoEncoderRockchip::Probe(VideoCodecType type) { if (device_accessible && format_supported) { capability.available = true; capability.detail = - "MPP encoder device and codec are available; compact I420 is copied into a " - "stride-aligned MPP buffer"; + "MPP encoder device and codec are available; RGA writes compact I420 into the " + "stride-aligned MPP DMA-BUF with an observable CPU fallback"; return capability; } @@ -218,6 +221,22 @@ bool VideoEncoderRockchip::OpenMpp() { CleanMpp(); return false; } + auto* frame_buffer_address = static_cast(mpp_buffer_get_ptr(state_->frame_buffer)); + if (!frame_buffer_address) { + LOG_WARN("{}", "MPP frame buffer is not CPU-addressable for fallback initialization"); + CleanMpp(); + return false; + } + mpp_buffer_sync_begin(state_->frame_buffer); + const size_t y_plane_size = state_->horizontal_stride * state_->vertical_stride; + std::memset(frame_buffer_address, 0, y_plane_size); + std::memset(frame_buffer_address + y_plane_size, 128, state_->frame_buffer_size - y_plane_size); + mpp_buffer_sync_end(state_->frame_buffer); + state_->rga_copy_in_available = + state_->frame_rga_handle.ImportFd(mpp_buffer_get_fd(state_->frame_buffer), state_->frame_buffer_size); + if (!state_->rga_copy_in_available) { + LOG_WARN("{}", "RGA could not import the MPP encoder DMA-BUF; CPU copy-in remains available"); + } ret = mpp_buffer_get(state_->buffer_group, &state_->packet_buffer, state_->frame_buffer_size); if (ret != MPP_OK) { LOG_WARN("MPP packet buffer allocation failed: {}", ret); @@ -357,21 +376,55 @@ VideoPacketPtr VideoEncoderRockchip::SendYUVFrame(void* data) { const size_t mpp_uv_stride = state_->horizontal_stride / 2; const size_t mpp_uv_height = state_->vertical_stride / 2; - mpp_buffer_sync_begin(state_->frame_buffer); - std::memset(destination, 0, mpp_y_size); - std::memset(destination + mpp_y_size, 128, state_->frame_buffer_size - mpp_y_size); - for (size_t row = 0; row < height_; ++row) { - std::memcpy(destination + row * state_->horizontal_stride, source + row * width_, width_); - } - auto* destination_u = destination + mpp_y_size; - auto* destination_v = destination_u + mpp_uv_stride * mpp_uv_height; - const auto* source_u = source + compact_y_size; - const auto* source_v = source_u + compact_uv_size; - for (size_t row = 0; row < height_ / 2; ++row) { - std::memcpy(destination_u + row * mpp_uv_stride, source_u + row * (width_ / 2), width_ / 2); - std::memcpy(destination_v + row * mpp_uv_stride, source_v + row * (width_ / 2), width_ / 2); + bool rga_copied = false; + if (state_->rga_copy_in_available) { + const auto rga_started = std::chrono::steady_clock::now(); + ScopedRgaBufferHandle source_handle(const_cast(source), + compact_y_size + compact_uv_size * 2); + IM_STATUS status = IM_STATUS_OUT_OF_MEMORY; + if (source_handle) { + auto source_image = wrapbuffer_handle_t(source_handle.Get(), static_cast(width_), + static_cast(height_), static_cast(width_), + static_cast(height_), RK_FORMAT_YCbCr_420_P); + auto target_image = + wrapbuffer_handle_t(state_->frame_rga_handle.Get(), static_cast(width_), + static_cast(height_), static_cast(state_->horizontal_stride), + static_cast(state_->vertical_stride), RK_FORMAT_YCbCr_420_P); + const im_rect source_rect{0, 0, static_cast(width_), static_cast(height_)}; + const im_rect target_rect = source_rect; + const im_rect empty_rect{}; + const rga_buffer_t empty_buffer{}; + status = improcess(source_image, target_image, empty_buffer, source_rect, target_rect, empty_rect, + IM_SYNC); + } + rga_copied = RockchipRgaSucceeded(status); + GetPreviewPipelineMetrics().RecordRgaOperation(rga_copied, ElapsedNanoseconds(rga_started)); + GetPreviewPipelineMetrics().RecordMppRgaCopyIn(rga_copied); + if (!rga_copied) { + LOG_WARN("MPP encoder RGA copy-in failed with status {} ({}); disabling it for this encoder", + status, imStrError_t(status)); + state_->rga_copy_in_available = false; + } + } + + if (!rga_copied) { + GetPreviewPipelineMetrics().RecordMppCpuCopyInFallback(); + mpp_buffer_sync_begin(state_->frame_buffer); + std::memset(destination, 0, mpp_y_size); + std::memset(destination + mpp_y_size, 128, state_->frame_buffer_size - mpp_y_size); + for (size_t row = 0; row < height_; ++row) { + std::memcpy(destination + row * state_->horizontal_stride, source + row * width_, width_); + } + auto* destination_u = destination + mpp_y_size; + auto* destination_v = destination_u + mpp_uv_stride * mpp_uv_height; + const auto* source_u = source + compact_y_size; + const auto* source_v = source_u + compact_uv_size; + for (size_t row = 0; row < height_ / 2; ++row) { + std::memcpy(destination_u + row * mpp_uv_stride, source_u + row * (width_ / 2), width_ / 2); + std::memcpy(destination_v + row * mpp_uv_stride, source_v + row * (width_ / 2), width_ / 2); + } + mpp_buffer_sync_end(state_->frame_buffer); } - mpp_buffer_sync_end(state_->frame_buffer); MppFrame frame = nullptr; auto ret = mpp_frame_init(&frame); @@ -458,6 +511,7 @@ void VideoEncoderRockchip::CleanMpp() { mpp_enc_cfg_deinit(state_->config); state_->config = nullptr; } + state_->frame_rga_handle.Reset(); if (state_->frame_buffer) { mpp_buffer_put(state_->frame_buffer); state_->frame_buffer = nullptr; diff --git a/src/media/VideoEncoderRockchip.h b/src/media/VideoEncoderRockchip.h index 0a61f7a49..6c1fe4820 100644 --- a/src/media/VideoEncoderRockchip.h +++ b/src/media/VideoEncoderRockchip.h @@ -9,11 +9,11 @@ namespace cosmo::media { class VideoEncoderCpu; struct RockchipEncoderState; -/// Rockchip MPP encoder with a bounded host-to-MPP copy boundary. +/// Shared Rockchip MPP encoder with an RGA-backed host-to-DMA boundary. /// /// The caller still owns compact I420 host memory. Each frame is copied into a -/// reusable, stride-aligned MPP buffer before hardware encoding. DMA-BUF -/// zero-copy ownership is intentionally outside this phase. +/// reusable, stride-aligned MPP DMA-BUF through RGA before hardware encoding. +/// A measured CPU fallback remains available for unsupported RGA layouts. class VideoEncoderRockchip final : public VideoEncoder { public: VideoEncoderRockchip(); diff --git a/src/media/VideoFrameProcRockchip.cc b/src/media/VideoFrameProcRockchip.cc index 9043890ed..ac88246c5 100644 --- a/src/media/VideoFrameProcRockchip.cc +++ b/src/media/VideoFrameProcRockchip.cc @@ -6,40 +6,12 @@ #include #include "media/PreviewPipelineMetrics.h" +#include "media/RockchipRgaBuffer.h" #include "util/Log.h" namespace cosmo::media { namespace { - class ScopedRgaHandle { - public: - ScopedRgaHandle(void* data, size_t size) { - if (data && size > 0 && size <= static_cast(std::numeric_limits::max())) { - handle_ = importbuffer_virtualaddr(data, static_cast(size)); - } - } - - ~ScopedRgaHandle() { - if (handle_ != 0) { - releasebuffer_handle(handle_); - } - } - - ScopedRgaHandle(const ScopedRgaHandle&) = delete; - ScopedRgaHandle& operator=(const ScopedRgaHandle&) = delete; - - [[nodiscard]] rga_buffer_handle_t Get() const { - return handle_; - } - - private: - rga_buffer_handle_t handle_{0}; - }; - - bool RgaSucceeded(IM_STATUS status) { - return status == IM_STATUS_SUCCESS || status == IM_STATUS_NOERROR; - } - uint64_t ElapsedNanoseconds(std::chrono::steady_clock::time_point started) { return static_cast( std::chrono::duration_cast(std::chrono::steady_clock::now() - started) @@ -99,8 +71,8 @@ VideoFramePtr VideoFrameProcRockchip::ConvertWithRga(const VideoFramePtr& frame, output->SetStreamIndex(frame->GetStreamIndex()); const auto started = std::chrono::steady_clock::now(); - ScopedRgaHandle src_handle(frame->GetData(), frame->GetSize()); - ScopedRgaHandle dst_handle(output->GetData(), output->GetSize()); + ScopedRgaBufferHandle src_handle(frame->GetData(), frame->GetSize()); + ScopedRgaBufferHandle dst_handle(output->GetData(), output->GetSize()); if (src_handle.Get() == 0 || dst_handle.Get() == 0) { const auto elapsed = ElapsedNanoseconds(started); GetPreviewPipelineMetrics().RecordRgaOperation(false, elapsed); @@ -112,8 +84,8 @@ VideoFramePtr VideoFrameProcRockchip::ConvertWithRga(const VideoFramePtr& frame, auto dst = wrapbuffer_handle_t(dst_handle.Get(), width, height, width, height, dst_rga_format); const auto status = imcvtcolor_t(src, dst, src_rga_format, dst_rga_format, color_mode, 1); const auto elapsed = ElapsedNanoseconds(started); - GetPreviewPipelineMetrics().RecordRgaOperation(RgaSucceeded(status), elapsed); - if (!RgaSucceeded(status)) { + GetPreviewPipelineMetrics().RecordRgaOperation(RockchipRgaSucceeded(status), elapsed); + if (!RockchipRgaSucceeded(status)) { LogFallbackOnce(operation, status); return nullptr; } @@ -139,8 +111,8 @@ VideoFramePtr VideoFrameProcRockchip::ResizeWithRga(const VideoFramePtr& frame, output->SetStreamIndex(frame->GetStreamIndex()); const auto started = std::chrono::steady_clock::now(); - ScopedRgaHandle src_handle(frame->GetData(), frame->GetSize()); - ScopedRgaHandle dst_handle(output->GetData(), output->GetSize()); + ScopedRgaBufferHandle src_handle(frame->GetData(), frame->GetSize()); + ScopedRgaBufferHandle dst_handle(output->GetData(), output->GetSize()); if (src_handle.Get() == 0 || dst_handle.Get() == 0) { const auto elapsed = ElapsedNanoseconds(started); GetPreviewPipelineMetrics().RecordRgaOperation(false, elapsed); @@ -154,8 +126,8 @@ VideoFramePtr VideoFrameProcRockchip::ResizeWithRga(const VideoFramePtr& frame, RK_FORMAT_YCbCr_420_P); const auto status = imresize_t(src, dst, 0.0, 0.0, INTER_LINEAR, 1); const auto elapsed = ElapsedNanoseconds(started); - GetPreviewPipelineMetrics().RecordRgaOperation(RgaSucceeded(status), elapsed); - if (!RgaSucceeded(status)) { + GetPreviewPipelineMetrics().RecordRgaOperation(RockchipRgaSucceeded(status), elapsed); + if (!RockchipRgaSucceeded(status)) { LogFallbackOnce(operation, status); return nullptr; } diff --git a/src/media/VideoFrameProcRockchip.h b/src/media/VideoFrameProcRockchip.h index b0b3ca46d..f0b4ec0dd 100644 --- a/src/media/VideoFrameProcRockchip.h +++ b/src/media/VideoFrameProcRockchip.h @@ -6,12 +6,12 @@ namespace cosmo::media { -/// Rockchip Copy-first frame processor. +/// General-purpose Rockchip RGA frame processor. /// -/// Frames remain owned by CosmoEdge's host memory pool. Each admitted RGA -/// operation imports the host buffers for the duration of one synchronous -/// operation and releases the handles before returning. This deliberately -/// does not introduce DMA-BUF ownership or zero-copy lifetime coupling. +/// Frames exposed through the generic VideoFrame API remain owned by +/// CosmoEdge's host memory pool. Each admitted RGA operation imports those +/// buffers for one synchronous operation. Detector and classifier fast paths +/// bypass this host-facing API and bind their MPP/RGA DMA-BUF inputs directly. class VideoFrameProcRockchip final : public VideoFrameProcCpu { public: explicit VideoFrameProcRockchip(IOsdTextRenderer& osd_service); diff --git a/src/nn/core/blob.h b/src/nn/core/blob.h index eb210d107..3d1a5bc73 100644 --- a/src/nn/core/blob.h +++ b/src/nn/core/blob.h @@ -11,6 +11,19 @@ namespace cosmo::nn { +enum class NativeImageColorSpace { + Unspecified = 0, + Bt601, + Bt709, + Bt2020, +}; + +enum class NativeImageColorRange { + Unspecified = 0, + Limited, + Full, +}; + struct PUBLIC BlobDesc { DeviceType device_type = DEVICE_NAIVE; @@ -40,6 +53,8 @@ struct PUBLIC BlobHandle { int width_stride{0}; int height_stride{0}; ImageFormat format{IMAGE_UNKNOWN}; + NativeImageColorSpace color_space{NativeImageColorSpace::Unspecified}; + NativeImageColorRange color_range{NativeImageColorRange::Unspecified}; [[nodiscard]] bool Valid() const { return fd >= 0 && bytes > 0 && width > 0 && height > 0 && width_stride >= width && diff --git a/src/nn/core/inference_pipeline_metrics.cc b/src/nn/core/inference_pipeline_metrics.cc index 38a840dc4..9f84689f8 100644 --- a/src/nn/core/inference_pipeline_metrics.cc +++ b/src/nn/core/inference_pipeline_metrics.cc @@ -96,6 +96,17 @@ void InferencePipelineMetrics::RecordRknnRgaResizeColor(uint64_t nanoseconds) { RecordStage(rknn_rga_resize_color_calls_, rknn_rga_resize_color_nanoseconds_, 1, nanoseconds); } +void InferencePipelineMetrics::RecordRknnRgaCropResize(uint64_t nanoseconds, bool success) { + RecordStage(rknn_rga_crop_resize_calls_, rknn_rga_crop_resize_nanoseconds_, 1, nanoseconds); + if (!success) + rknn_rga_crop_resize_failures_.fetch_add(1, std::memory_order_relaxed); +} + +void InferencePipelineMetrics::RecordRknnRgaCropSource(bool dmabuf) { + auto& counter = dmabuf ? rknn_rga_crop_dmabuf_frames_ : rknn_rga_crop_host_fallbacks_; + counter.fetch_add(1, std::memory_order_relaxed); +} + void InferencePipelineMetrics::RecordRknnRgaFailure() { rknn_rga_failures_.fetch_add(1, std::memory_order_relaxed); } @@ -104,6 +115,11 @@ void InferencePipelineMetrics::RecordRknnCpuResizeFallback(uint64_t nanoseconds) RecordStage(rknn_cpu_resize_fallback_calls_, rknn_cpu_resize_fallback_nanoseconds_, 1, nanoseconds); } +void InferencePipelineMetrics::RecordRknnCpuCropResizeFallback(uint64_t nanoseconds) { + RecordStage(rknn_cpu_crop_resize_fallback_calls_, rknn_cpu_crop_resize_fallback_nanoseconds_, 1, + nanoseconds); +} + void InferencePipelineMetrics::RecordRknnCpuNormalizeFallback(uint64_t nanoseconds) { RecordStage(rknn_cpu_normalize_fallback_calls_, rknn_cpu_normalize_fallback_nanoseconds_, 1, nanoseconds); } @@ -284,9 +300,16 @@ InferencePipelineMetricsSnapshot InferencePipelineMetrics::Snapshot() const { SNAPSHOT_FIELD(rknn_rga_fill_nanoseconds); SNAPSHOT_FIELD(rknn_rga_resize_color_calls); SNAPSHOT_FIELD(rknn_rga_resize_color_nanoseconds); + SNAPSHOT_FIELD(rknn_rga_crop_resize_calls); + SNAPSHOT_FIELD(rknn_rga_crop_resize_nanoseconds); + SNAPSHOT_FIELD(rknn_rga_crop_resize_failures); + SNAPSHOT_FIELD(rknn_rga_crop_dmabuf_frames); + SNAPSHOT_FIELD(rknn_rga_crop_host_fallbacks); SNAPSHOT_FIELD(rknn_rga_failures); SNAPSHOT_FIELD(rknn_cpu_resize_fallback_calls); SNAPSHOT_FIELD(rknn_cpu_resize_fallback_nanoseconds); + SNAPSHOT_FIELD(rknn_cpu_crop_resize_fallback_calls); + SNAPSHOT_FIELD(rknn_cpu_crop_resize_fallback_nanoseconds); SNAPSHOT_FIELD(rknn_cpu_normalize_fallback_calls); SNAPSHOT_FIELD(rknn_cpu_normalize_fallback_nanoseconds); SNAPSHOT_FIELD(rknn_native_input_map_calls); diff --git a/src/nn/core/inference_pipeline_metrics.h b/src/nn/core/inference_pipeline_metrics.h index 756bb1963..15a775728 100644 --- a/src/nn/core/inference_pipeline_metrics.h +++ b/src/nn/core/inference_pipeline_metrics.h @@ -60,9 +60,16 @@ struct InferencePipelineMetricsSnapshot { uint64_t rknn_rga_fill_nanoseconds{0}; uint64_t rknn_rga_resize_color_calls{0}; uint64_t rknn_rga_resize_color_nanoseconds{0}; + uint64_t rknn_rga_crop_resize_calls{0}; + uint64_t rknn_rga_crop_resize_nanoseconds{0}; + uint64_t rknn_rga_crop_resize_failures{0}; + uint64_t rknn_rga_crop_dmabuf_frames{0}; + uint64_t rknn_rga_crop_host_fallbacks{0}; uint64_t rknn_rga_failures{0}; uint64_t rknn_cpu_resize_fallback_calls{0}; uint64_t rknn_cpu_resize_fallback_nanoseconds{0}; + uint64_t rknn_cpu_crop_resize_fallback_calls{0}; + uint64_t rknn_cpu_crop_resize_fallback_nanoseconds{0}; uint64_t rknn_cpu_normalize_fallback_calls{0}; uint64_t rknn_cpu_normalize_fallback_nanoseconds{0}; uint64_t rknn_native_input_map_calls{0}; @@ -139,8 +146,11 @@ class InferencePipelineMetrics { void RecordRknnPreprocessFastHit(); void RecordRknnRgaFill(uint64_t nanoseconds); void RecordRknnRgaResizeColor(uint64_t nanoseconds); + void RecordRknnRgaCropResize(uint64_t nanoseconds, bool success); + void RecordRknnRgaCropSource(bool dmabuf); void RecordRknnRgaFailure(); void RecordRknnCpuResizeFallback(uint64_t nanoseconds); + void RecordRknnCpuCropResizeFallback(uint64_t nanoseconds); void RecordRknnCpuNormalizeFallback(uint64_t nanoseconds); void RecordRknnNativeInputMap(uint64_t nanoseconds); void RecordRknnInputFormat(bool native_int8, bool compatibility_fallback = false, @@ -220,9 +230,16 @@ class InferencePipelineMetrics { std::atomic rknn_rga_fill_nanoseconds_{0}; std::atomic rknn_rga_resize_color_calls_{0}; std::atomic rknn_rga_resize_color_nanoseconds_{0}; + std::atomic rknn_rga_crop_resize_calls_{0}; + std::atomic rknn_rga_crop_resize_nanoseconds_{0}; + std::atomic rknn_rga_crop_resize_failures_{0}; + std::atomic rknn_rga_crop_dmabuf_frames_{0}; + std::atomic rknn_rga_crop_host_fallbacks_{0}; std::atomic rknn_rga_failures_{0}; std::atomic rknn_cpu_resize_fallback_calls_{0}; std::atomic rknn_cpu_resize_fallback_nanoseconds_{0}; + std::atomic rknn_cpu_crop_resize_fallback_calls_{0}; + std::atomic rknn_cpu_crop_resize_fallback_nanoseconds_{0}; std::atomic rknn_cpu_normalize_fallback_calls_{0}; std::atomic rknn_cpu_normalize_fallback_nanoseconds_{0}; std::atomic rknn_native_input_map_calls_{0}; diff --git a/src/nn/core/shared_resource.h b/src/nn/core/shared_resource.h index 7bf4fd1b5..7438960cd 100644 --- a/src/nn/core/shared_resource.h +++ b/src/nn/core/shared_resource.h @@ -114,6 +114,10 @@ class SharedResource { RknnBoundInputProvider* rknn_bound_input_provider{nullptr}; RknnBoundInputTarget rknn_bound_input_target{}; + // Set by the graph's normalize node after shape inference. Producers may + // write directly into the RKNN input DMA-BUF only when the complete + // preprocessing contract (RGB UINT8, 0..255 -> 0..1) is compatible. + bool rknn_bound_input_preprocess_compatible{false}; }; } // namespace cosmo::nn diff --git a/src/nn/device/cpu/cpu_crop_resize_node.h b/src/nn/device/cpu/cpu_crop_resize_node.h index 57716f385..8711575d0 100644 --- a/src/nn/device/cpu/cpu_crop_resize_node.h +++ b/src/nn/device/cpu/cpu_crop_resize_node.h @@ -31,7 +31,7 @@ class CpuCropResizeNode : public Node { Status Forward(std::vector>& bottom_blobs, std::vector>& top_blobs) override; -private: +protected: Status PrepareRect(std::shared_ptr host_rect, int image_w, int image_h); static void BilinearResize(const uint8_t* src, int src_w, int src_h, int channels, uint8_t* dst, diff --git a/src/nn/device/rknn/rknn_device.cc b/src/nn/device/rknn/rknn_device.cc index 3a2b8c7db..fe018c965 100644 --- a/src/nn/device/rknn/rknn_device.cc +++ b/src/nn/device/rknn/rknn_device.cc @@ -4,9 +4,10 @@ namespace cosmo::nn { -// RKNN's copy-first backend uses graph-owned host buffers. Registering a -// DEVICE_RKNN allocator keeps externally wrapped image blobs and BlobStore's -// calculate-device lifecycle valid without pretending the buffers are NPU DMA. +// Graph metadata and compatibility tensors retain the graph-owned NaiveDevice +// lifecycle. Native input allocation and rknn_set_io_mem binding are owned by +// RknnNetNode/RKNN preprocess, so this registration does not imply host-copy +// inference on the admitted DMA-BUF path. TypeDeviceRegister g_rknn_device_register(DEVICE_RKNN); } // namespace cosmo::nn diff --git a/src/nn/device/rknn/rknn_net_node.cc b/src/nn/device/rknn/rknn_net_node.cc index c18caed42..71a85da99 100644 --- a/src/nn/device/rknn/rknn_net_node.cc +++ b/src/nn/device/rknn/rknn_net_node.cc @@ -14,6 +14,10 @@ #include #include +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif + #include "nn/core/inference_pipeline_metrics.h" #include "nn/device/rknn/rknn_yolov8_adapter.h" #include "nn/node/node_type_utils.h" @@ -180,6 +184,13 @@ rknn_core_mask ResolveRknnCoreMask(RknnCoreMode mode, uint64_t context_sequence) } } +bool ShouldConfigureRknnCoreMask(RknnCoreMode mode) { + // Leaving the runtime default untouched is the portable automatic mode. + // Single-core RKNPU2 devices reject even RKNN_NPU_CORE_AUTO, while + // multi-core targets still accept the explicit core/split modes below. + return mode != RknnCoreMode::Auto; +} + const char* RknnCoreModeName(RknnCoreMode mode) { switch (mode) { case RknnCoreMode::Core0: @@ -291,24 +302,6 @@ bool IsRknnRgaBoundInputCompatible(const rknn_tensor_attr& attr, int height, int return IsRknnBoundInt8InputCompatible(attr, desc, reason); } -bool ConfigureRknnRgaUint8InputAttr(const rknn_tensor_attr& native_attr, int height, int width, - rknn_tensor_attr& bound_attr, std::string* reason) { - if (!IsRknnRgaBoundInputCompatible(native_attr, height, width, reason)) - return false; - - // RKNN's native tensor is INT8, but the zero-copy contract permits a UINT8 - // bound input. Runtime then fuses the model's normalize/quantize operation - // onto the NPU, so RGA can write RGB bytes directly without a host-wide - // XOR/subtract-128 pass. This mirrors Rockchip's rknpu2 zero-copy sample. - bound_attr = native_attr; - bound_attr.type = RKNN_TENSOR_UINT8; - bound_attr.fmt = RKNN_TENSOR_NHWC; - bound_attr.pass_through = 0; - if (reason) - reason->clear(); - return true; -} - bool CopyRknnPackedInt8Input(const int8_t* source, size_t source_bytes, int8_t* destination, size_t destination_bytes, int height, int width, int channels, int width_stride, std::string* reason) { @@ -409,18 +402,44 @@ bool RequantizeRknnPackedUint8ToInt8InPlace(uint8_t* data, size_t data_bytes, in const uint64_t required = static_cast(height) * effective_stride * channels; if (required == 0 || required > data_bytes) return reject("RGA bound input requantization buffer is smaller than its tensor contract"); + const auto flip_sign_bits = [](uint8_t* bytes, size_t count) { + size_t index = 0; +#if defined(__ARM_NEON) || defined(__ARM_NEON__) + const uint8x16_t sign_bit = vdupq_n_u8(0x80); + for (; index + 64 <= count; index += 64) { + vst1q_u8(bytes + index, veorq_u8(vld1q_u8(bytes + index), sign_bit)); + vst1q_u8(bytes + index + 16, veorq_u8(vld1q_u8(bytes + index + 16), sign_bit)); + vst1q_u8(bytes + index + 32, veorq_u8(vld1q_u8(bytes + index + 32), sign_bit)); + vst1q_u8(bytes + index + 48, veorq_u8(vld1q_u8(bytes + index + 48), sign_bit)); + } + for (; index + 16 <= count; index += 16) + vst1q_u8(bytes + index, veorq_u8(vld1q_u8(bytes + index), sign_bit)); +#endif + for (; index < count; ++index) + bytes[index] ^= 0x80; + }; + const size_t row_bytes = static_cast(width) * static_cast(channels); const size_t stride_bytes = static_cast(effective_stride) * static_cast(channels); - for (int row = 0; row < height; ++row) { - auto* row_data = data + static_cast(row) * stride_bytes; - for (size_t index = 0; index < row_bytes; ++index) - row_data[index] ^= 0x80; + if (row_bytes == stride_bytes) { + flip_sign_bits(data, static_cast(height) * row_bytes); + } else { + for (int row = 0; row < height; ++row) + flip_sign_bits(data + static_cast(row) * stride_bytes, row_bytes); } if (reason) reason->clear(); return true; } +const char* RknnRgaBoundRequantizeImplementation() { +#if defined(__ARM_NEON) || defined(__ARM_NEON__) + return "arm-neon-xor-sign-bit"; +#else + return "scalar-xor-sign-bit"; +#endif +} + bool RknnFastOutputEnabled() { return EnvironmentFlag("COSMO_RKNN_FAST_OUTPUT", true); } @@ -504,7 +523,7 @@ bool RknnNetNode::AllocateAndBindInputMemory(rknn_tensor_attr attr, BoundInputMo bound_input_attr_ = attr; bound_input_memory_ = memory; bound_input_mode_ = mode; - if (mode == BoundInputMode::RgaUint8 || mode == BoundInputMode::RgaNativeInt8) + if (mode == BoundInputMode::RgaNativeInt8) PublishRgaBoundInputTarget(); reason.clear(); return true; @@ -535,37 +554,17 @@ bool RknnNetNode::TryBindRgaInputMemory(int height, int width, std::string& reas if (!IsRknnRgaBoundInputCompatible(native_attr, height, width, &reason)) return false; - if (IsRknnRgbUint8InputContract(input_contract_)) { - rknn_tensor_attr uint8_attr{}; - if (!ConfigureRknnRgaUint8InputAttr(native_attr, height, width, uint8_attr, &reason)) - return false; - std::string uint8_reason; - if (AllocateAndBindInputMemory(uint8_attr, BoundInputMode::RgaUint8, uint8_reason)) - return true; - - // Older or model-specific runtimes may reject the declared UINT8 - // contract. Preserve the native-INT8 binding as a correctness fallback; - // telemetry keeps this CPU-requantized mode visible. - native_attr.pass_through = 1; - if (AllocateAndBindInputMemory(native_attr, BoundInputMode::RgaNativeInt8, reason)) { - LOG_WARN("RKNN fused UINT8 bound input unavailable ({}); using native INT8 fallback", - uint8_reason); - return true; - } - reason = "UINT8 bound input failed: " + uint8_reason + "; native INT8 fallback failed: " + reason; - return false; - } - - // A native INT8 tensor does not prove that the model embeds normalization. - // Host-owned models therefore retain the exact, already-qualified - // uint8-to-int8 conversion until their config declares the contract. + // RGA implementations do not universally expose NN quantization. Bind the + // model's native INT8 tensor and let RGA write RGB bytes into that DMA-BUF; + // Forward then performs the one mathematically required sign-bit transform + // in place. This keeps resize/color/crop and the image-sized copy off CPU + // while preserving the exact u8/255 -> int8(zp=-128) model contract. native_attr.pass_through = 1; return AllocateAndBindInputMemory(native_attr, BoundInputMode::RgaNativeInt8, reason); } void RknnNetNode::PublishRgaBoundInputTarget() { - const bool rga_bound_mode = - bound_input_mode_ == BoundInputMode::RgaUint8 || bound_input_mode_ == BoundInputMode::RgaNativeInt8; + const bool rga_bound_mode = bound_input_mode_ == BoundInputMode::RgaNativeInt8; if (!shared_resource || !bound_input_memory_ || !rga_bound_mode) { return; } @@ -594,8 +593,8 @@ void RknnNetNode::ClearRgaBoundInputTarget() { bool RknnNetNode::EnsureRgaBoundInput(int height, int width, std::string& reason) { std::lock_guard lock(mutex_); - if (context_ == 0 || !detector_model_) { - reason = "RKNN detector context is not initialized"; + if (context_ == 0 || !IsRknnRgbUint8InputContract(input_contract_)) { + reason = "RKNN RGB UINT8 input context is not initialized"; return false; } if (!RknnBoundInputEnabled() || !RknnRgaBoundInputEnabled()) { @@ -603,8 +602,7 @@ bool RknnNetNode::EnsureRgaBoundInput(int height, int width, std::string& reason return false; } if (bound_input_memory_) { - const bool rga_bound_mode = bound_input_mode_ == BoundInputMode::RgaUint8 || - bound_input_mode_ == BoundInputMode::RgaNativeInt8; + const bool rga_bound_mode = bound_input_mode_ == BoundInputMode::RgaNativeInt8; if (rga_bound_mode && shared_resource && shared_resource->rknn_bound_input_target.owner == this && shared_resource->rknn_bound_input_target.Matches(height, width)) { reason.clear(); @@ -623,9 +621,11 @@ bool RknnNetNode::EnsureRgaBoundInput(int height, int width, std::string& reason rga_bound_input_eligible_ = false; return false; } - LOG_INFO("RKNN RGA input bound: mode={} bytes={} fd={} width_stride={}", - bound_input_mode_ == BoundInputMode::RgaUint8 ? "fused-uint8" : "native-int8", - bound_input_memory_->size, bound_input_memory_->fd, bound_input_attr_.w_stride); + LOG_INFO( + "RKNN RGA input bound: mode=native-int8+sign-bit-transform implementation={} bytes={} fd={} " + "width_stride={}", + RknnRgaBoundRequantizeImplementation(), bound_input_memory_->size, bound_input_memory_->fd, + bound_input_attr_.w_stride); return true; } @@ -739,11 +739,14 @@ Status RknnNetNode::LoadWeight(const char* data, size_t size) { if (!core_mode_valid) { LOG_WARN("Invalid COSMO_RKNN_CORE_MODE value:{}, fallback:auto", core_mode_env ? core_mode_env : ""); } - const auto core_mask = ResolveRknnCoreMask(core_mode, context_sequence); - result = rknn_set_core_mask(context_, core_mask); - if (result != RKNN_SUCC) { - DestroyContext(); - return RknnError("rknn_set_core_mask", result); + const auto core_mask = ResolveRknnCoreMask(core_mode, context_sequence); + const bool configure_core_mask = ShouldConfigureRknnCoreMask(core_mode); + if (configure_core_mask) { + result = rknn_set_core_mask(context_, core_mask); + if (result != RKNN_SUCC) { + DestroyContext(); + return RknnError("rknn_set_core_mask", result); + } } rknn_sdk_version version{}; @@ -766,15 +769,22 @@ Status RknnNetNode::LoadWeight(const char* data, size_t size) { DestroyContext(); return Status(COSMO_NN_ERR_INVALID_CFG, "RKNN logical output count does not match config.json"); } - if (detector_model_ && shared_resource) + // Publish the graph-local capability endpoint independently of admission. + // EnsureRgaBoundInput remains the single authority for the model contract, + // runtime tensor attributes, and allocation/binding decision. Conditional + // publication hides the rejection reason from an otherwise compatible + // producer and lets the later generic copy path win silently. + if (shared_resource) shared_resource->rknn_bound_input_provider = this; LOG_INFO( "RKNN model loaded: api={} driver={} inputs={} runtime_outputs={} logical_outputs={} " - "output_adapter={} native_int8_output={} core_mode={} core_mask={} context_sequence={}", + "output_adapter={} native_int8_output={} rgb_uint8_input_contract={} core_mode={} core_mask={} " + "core_mask_applied={} context_sequence={}", version.api_version, version.drv_version, io_count_.n_input, io_count_.n_output, logical_outputs, RknnOutputAdapterName(output_adapter_contract_.kind), native_yolov8_outputs_, - RknnCoreModeName(core_mode), static_cast(core_mask), context_sequence); + IsRknnRgbUint8InputContract(input_contract_), RknnCoreModeName(core_mode), + static_cast(core_mask), configure_core_mask, context_sequence); return COSMO_NN_OK; } @@ -918,9 +928,8 @@ Status RknnNetNode::Forward(std::vector>& bottom_blobs, shared_resource->rknn_bound_input_target.owner == this) { auto& target = shared_resource->rknn_bound_input_target; target.frame_ready = false; - const bool rga_bound_mode = bound_input_mode_ == BoundInputMode::RgaUint8 || - bound_input_mode_ == BoundInputMode::RgaNativeInt8; - const bool blob_matches = input_desc.data_type == DATA_TYPE_INT8 && + const bool rga_bound_mode = bound_input_mode_ == BoundInputMode::RgaNativeInt8; + const bool blob_matches = input_desc.data_type == DATA_TYPE_INT8 && input_desc.data_format == DATA_FORMAT_NHWC && input_desc.image_format == IMAGE_RGB && input_desc.dims.size() == 4 && input_desc.dims[0] == 1 && input_desc.dims[1] == target.height && @@ -1044,9 +1053,7 @@ Status RknnNetNode::Forward(std::vector>& bottom_blobs, LOG_WARN("RKNN bound input unavailable; retaining rknn_inputs_set path: {}", bind_reason); } } - const bool runtime_uint8_contract = - uint8_contract_input || - (native_int8 && bound_input_memory_ && bound_input_mode_ == BoundInputMode::RgaUint8); + const bool runtime_uint8_contract = uint8_contract_input; if (!rga_bound_frame) GetInferencePipelineMetrics().RecordRknnInputFormat(native_int8 && !runtime_uint8_contract, compatibility_fallback, runtime_uint8_contract); @@ -1055,17 +1062,7 @@ Status RknnNetNode::Forward(std::vector>& bottom_blobs, const auto copy_started = MetricsClock::now(); const size_t source_bytes = input.size; bool copied = false; - if (bound_input_mode_ == BoundInputMode::RgaUint8 && native_int8) { - copied = CopyRknnPackedNativeInt8ToUint8( - static_cast(input.buf), source_bytes, - static_cast(bound_input_memory_->virt_addr), bound_input_memory_->size, - input_height, input_width, 3, static_cast(bound_input_attr_.w_stride), ©_reason); - } else if (bound_input_mode_ == BoundInputMode::RgaUint8 && uint8_contract_input) { - copied = CopyRknnPackedInt8Input(reinterpret_cast(input.buf), source_bytes, - static_cast(bound_input_memory_->virt_addr), - bound_input_memory_->size, input_height, input_width, 3, - static_cast(bound_input_attr_.w_stride), ©_reason); - } else if (native_int8) { + if (native_int8) { copied = CopyRknnPackedInt8Input(static_cast(input.buf), source_bytes, static_cast(bound_input_memory_->virt_addr), bound_input_memory_->size, input_height, input_width, 3, @@ -1086,7 +1083,7 @@ Status RknnNetNode::Forward(std::vector>& bottom_blobs, GetInferencePipelineMetrics().RecordRknnBoundInputFrame(); use_bound_input = true; } - if (rga_bound_frame && bound_input_mode_ == BoundInputMode::RgaNativeInt8) { + if (rga_bound_frame) { const auto sync_from_started = MetricsClock::now(); int sync_result = rknn_mem_sync(context_, bound_input_memory_, RKNN_MEMORY_SYNC_FROM_DEVICE); GetInferencePipelineMetrics().RecordRknnBoundInputSync(ElapsedNanoseconds(sync_from_started), @@ -1111,10 +1108,8 @@ Status RknnNetNode::Forward(std::vector>& bottom_blobs, if (sync_result != RKNN_SUCC) return finish(RknnError("rknn_mem_sync to device", sync_result)); } - if (rga_bound_frame) { - GetInferencePipelineMetrics().RecordRknnRgaBoundInputFrame(bound_input_mode_ == - BoundInputMode::RgaUint8); - } + if (rga_bound_frame) + GetInferencePipelineMetrics().RecordRknnRgaBoundInputFrame(false); int result = RKNN_SUCC; if (!use_bound_input) { bound_input_eligible_ = false; diff --git a/src/nn/device/rknn/rknn_net_node.h b/src/nn/device/rknn/rknn_net_node.h index d71bfa80e..985d44ab5 100644 --- a/src/nn/device/rknn/rknn_net_node.h +++ b/src/nn/device/rknn/rknn_net_node.h @@ -23,8 +23,6 @@ bool IsRknnBoundInt8InputCompatible(const rknn_tensor_attr& attr, const BlobDesc std::string* reason = nullptr); bool IsRknnRgaBoundInputCompatible(const rknn_tensor_attr& attr, int height, int width, std::string* reason = nullptr); -bool ConfigureRknnRgaUint8InputAttr(const rknn_tensor_attr& native_attr, int height, int width, - rknn_tensor_attr& bound_attr, std::string* reason = nullptr); bool CopyRknnPackedInt8Input(const int8_t* source, size_t source_bytes, int8_t* destination, size_t destination_bytes, int height, int width, int channels, int width_stride, std::string* reason = nullptr); @@ -35,6 +33,7 @@ bool ConvertRknnNormalizedFloatToUint8(const float* source, size_t source_count, size_t destination_count, std::string* reason = nullptr); bool RequantizeRknnPackedUint8ToInt8InPlace(uint8_t* data, size_t data_bytes, int height, int width, int channels, int width_stride, std::string* reason = nullptr); +const char* RknnRgaBoundRequantizeImplementation(); bool RknnFastOutputEnabled(); bool RknnDirectCandidatesEnabled(); bool RknnBoundInputEnabled(); @@ -50,6 +49,7 @@ enum class RknnCoreMode : uint8_t { RknnCoreMode ParseRknnCoreMode(const std::string& value, bool* valid = nullptr); rknn_core_mask ResolveRknnCoreMask(RknnCoreMode mode, uint64_t context_sequence); +bool ShouldConfigureRknnCoreMask(RknnCoreMode mode); const char* RknnCoreModeName(RknnCoreMode mode); class RknnNetNode final : public NetNode, public RknnBoundInputProvider { @@ -71,7 +71,6 @@ class RknnNetNode final : public NetNode, public RknnBoundInputProvider { enum class BoundInputMode : uint8_t { None = 0, NativeInt8, - RgaUint8, RgaNativeInt8, }; diff --git a/src/nn/device/rknn/rknn_node_creator.cc b/src/nn/device/rknn/rknn_node_creator.cc index 09d72d096..4b959d832 100644 --- a/src/nn/device/rknn/rknn_node_creator.cc +++ b/src/nn/device/rknn/rknn_node_creator.cc @@ -19,6 +19,8 @@ std::unique_ptr RknnNodeCreator::CreateNode(NodeType type) { if (RknnFastPreprocessEnabled()) { if (type == NODE_RESIZE) return std::make_unique(); + if (type == NODE_CROP_RESIZE) + return std::make_unique(); if (type == NODE_NORMALIZE) return std::make_unique(); } diff --git a/src/nn/device/rknn/rknn_preprocess_node.cc b/src/nn/device/rknn/rknn_preprocess_node.cc index 8e19f1011..c52ba1d53 100644 --- a/src/nn/device/rknn/rknn_preprocess_node.cc +++ b/src/nn/device/rknn/rknn_preprocess_node.cc @@ -16,6 +16,7 @@ #include #include +#include "media/RockchipRgaBuffer.h" #include "nn/core/inference_pipeline_metrics.h" #include "nn/device/rknn/rknn_net_node.h" #include "nn/node/node_type_utils.h" @@ -29,6 +30,9 @@ namespace { constexpr int kDetectorInputSize = 640; constexpr float kNormalizeScale = 0.00392157f; + // RGA2-class cores support at least 1/8..8 scaling. Keep the shared path + // conservative so the same implementation is valid on both RGA2 and RGA3. + constexpr int kConservativeRgaScaleLimit = 8; uint64_t ElapsedNanoseconds(MetricsClock::time_point started_at) { return static_cast( @@ -49,47 +53,6 @@ namespace { return default_value; } - bool RgaSucceeded(IM_STATUS status) { - return status == IM_STATUS_SUCCESS || status == IM_STATUS_NOERROR; - } - - class ScopedRgaHandle { - public: - ScopedRgaHandle() = default; - - ScopedRgaHandle(void* data, size_t size) { - ImportVirtual(data, size); - } - - ~ScopedRgaHandle() { - if (handle_ != 0) - releasebuffer_handle(handle_); - } - - [[nodiscard]] rga_buffer_handle_t Get() const { - return handle_; - } - - void ImportVirtual(void* data, size_t size) { - if (handle_ != 0 || !data || size == 0 || - size > static_cast(std::numeric_limits::max())) { - return; - } - handle_ = importbuffer_virtualaddr(data, static_cast(size)); - } - - void ImportFd(int fd, size_t size) { - if (handle_ != 0 || fd < 0 || size == 0 || - size > static_cast(std::numeric_limits::max())) { - return; - } - handle_ = importbuffer_fd(fd, static_cast(size)); - } - - private: - rga_buffer_handle_t handle_{0}; - }; - void LogRgaFallbackOnce(IM_STATUS status) { static std::atomic_flag logged = ATOMIC_FLAG_INIT; if (!logged.test_and_set(std::memory_order_relaxed)) { @@ -113,6 +76,23 @@ namespace { } } + void LogRgaCropResizeFallbackOnce(IM_STATUS status) { + static std::atomic_flag logged = ATOMIC_FLAG_INIT; + if (!logged.test_and_set(std::memory_order_relaxed)) { + LOG_WARN("RKNN RGA crop-resize failed with status {} ({}); using CPU fallback", status, + imStrError_t(status)); + } + } + + void LogRgaBt2020DowngradeOnce() { + static std::atomic_flag logged = ATOMIC_FLAG_INIT; + if (!logged.test_and_set(std::memory_order_relaxed)) { + LOG_WARN( + "This librga header/runtime does not expose BT.2020 full-CSC modes; " + "using the matching BT.709 range for this source"); + } + } + void BilinearResizePacked(const uint8_t* source, int source_width, int source_height, int channels, uint8_t* destination, int destination_width, int destination_height, bool swap_red_blue) { @@ -154,6 +134,135 @@ namespace { return pixels * 3; } + IM_COLOR_SPACE_MODE ResolveRgaYuvColorSpace(const BlobHandle::NativeImage& image) { + const bool full_range = image.color_range == NativeImageColorRange::Full; + switch (image.color_space) { + case NativeImageColorSpace::Bt709: + return full_range ? IM_YUV_BT709_FULL_RANGE : IM_YUV_BT709_LIMIT_RANGE; + case NativeImageColorSpace::Bt2020: + if (!media::RockchipRgaHasBt2020ColorSpace()) + LogRgaBt2020DowngradeOnce(); + return media::RockchipRgaBt2020ColorSpace(full_range); + case NativeImageColorSpace::Bt601: + case NativeImageColorSpace::Unspecified: + default: + return full_range ? IM_YUV_BT601_FULL_RANGE : IM_YUV_BT601_LIMIT_RANGE; + } + } + + im_rect AlignYuv420Crop(int x, int y, int width, int height, int image_width, int image_height) { + const int aligned_image_width = image_width & ~1; + const int aligned_image_height = image_height & ~1; + const int left = std::max(0, x & ~1); + const int top = std::max(0, y & ~1); + int right = std::min(aligned_image_width, (x + width + 1) & ~1); + int bottom = std::min(aligned_image_height, (y + height + 1) & ~1); + if (right <= left) + right = std::min(aligned_image_width, left + 2); + if (bottom <= top) + bottom = std::min(aligned_image_height, top + 2); + return {left, top, right - left, bottom - top}; + } + + int NextRgaScaleDimension(int current, int target) { + if (current <= 0 || target <= 0) + return 0; + const auto current_wide = static_cast(current); + const auto target_wide = static_cast(target); + if (target_wide > current_wide * kConservativeRgaScaleLimit) { + return static_cast(current_wide * kConservativeRgaScaleLimit); + } + if (current_wide > target_wide * kConservativeRgaScaleLimit) { + return static_cast((current_wide + kConservativeRgaScaleLimit - 1) / + kConservativeRgaScaleLimit); + } + return target; + } + + bool RunStagedRgaCropResize(rga_buffer_t source, const im_rect& source_rect, + IM_COLOR_SPACE_MODE source_color_space, rga_buffer_t target, + const im_rect& target_rect, IM_STATUS& last_status) { + if (source_rect.width <= 0 || source_rect.height <= 0 || target_rect.width <= 0 || + target_rect.height <= 0) { + last_status = IM_STATUS_INVALID_PARAM; + return false; + } + + std::vector> destinations; + int current_width = source_rect.width; + int current_height = source_rect.height; + while (true) { + const int next_width = NextRgaScaleDimension(current_width, target_rect.width); + const int next_height = NextRgaScaleDimension(current_height, target_rect.height); + if (next_width <= 0 || next_height <= 0 || destinations.size() >= 16) { + last_status = IM_STATUS_INVALID_PARAM; + return false; + } + destinations.emplace_back(next_width, next_height); + if (next_width == target_rect.width && next_height == target_rect.height) + break; + if (next_width == current_width && next_height == current_height) { + last_status = IM_STATUS_INVALID_PARAM; + return false; + } + current_width = next_width; + current_height = next_height; + } + + std::vector> stage_storage; + std::vector stage_handles; + std::vector stage_buffers; + if (destinations.size() > 1) { + const auto intermediate_count = destinations.size() - 1; + stage_storage.reserve(intermediate_count); + stage_handles.reserve(intermediate_count); + stage_buffers.reserve(intermediate_count); + } + + rga_buffer_t current_source = source; + im_rect current_source_rect = source_rect; + for (size_t index = 0; index < destinations.size(); ++index) { + const bool final_pass = index + 1 == destinations.size(); + rga_buffer_t current_target{}; + im_rect current_target_rect{}; + if (final_pass) { + current_target = target; + current_target_rect = target_rect; + } else { + const int width = destinations[index].first; + const int height = destinations[index].second; + const size_t bytes = PackedByteCount(width, height); + stage_storage.emplace_back(bytes); + stage_handles.emplace_back(); + if (bytes == 0 || !stage_handles.back().ImportVirtual(stage_storage.back().data(), bytes)) { + last_status = IM_STATUS_OUT_OF_MEMORY; + return false; + } + stage_buffers.push_back(wrapbuffer_handle_t(stage_handles.back().Get(), width, height, width, + height, RK_FORMAT_RGB_888)); + current_target = stage_buffers.back(); + current_target_rect = {0, 0, width, height}; + } + + if (index == 0 && source_color_space != IM_COLOR_SPACE_DEFAULT) { + media::SetRgaYuvToRgbColorSpace(current_source, current_target, source_color_space); + } + const im_rect empty_rect{}; + const rga_buffer_t empty_buffer{}; + const auto started = MetricsClock::now(); + last_status = improcess(current_source, current_target, empty_buffer, current_source_rect, + current_target_rect, empty_rect, IM_SYNC); + const bool success = media::RockchipRgaSucceeded(last_status); + GetInferencePipelineMetrics().RecordRknnRgaCropResize(ElapsedNanoseconds(started), success); + if (!success) + return false; + + current_source = current_target; + current_source_rect = {0, 0, current_target_rect.width, current_target_rect.height}; + } + return true; + } + } // namespace bool RknnFastPreprocessEnabled() { @@ -181,8 +290,8 @@ bool IsRknnDetectorResizeContract(int out_height, int out_width, int gravity, bool IsRknnNativeNormalizeContract(const std::vector& mean, const std::vector& std_dev, float scale, const DimsVector& input_dims) { - if (input_dims.size() != 4 || input_dims[0] <= 0 || input_dims[1] != kDetectorInputSize || - input_dims[2] != kDetectorInputSize || input_dims[3] != 3 || mean.size() < 3 || !std_dev.empty() || + if (input_dims.size() != 4 || input_dims[0] != 1 || input_dims[1] <= 0 || input_dims[2] <= 0 || + input_dims[3] != 3 || mean.size() < 3 || !std_dev.empty() || std::fabs(scale - kNormalizeScale) > 1e-8f) { return false; } @@ -257,9 +366,19 @@ void RknnResizeNode::ReleaseRgaBoundTarget() { } bool RknnResizeNode::AcquireRgaBoundTarget(uint32_t& handle) { - handle = 0; - if (!detector_contract_ || rga_bound_target_unavailable_ || !RknnRgaBoundInputEnabled() || - !shared_resource || !shared_resource->rknn_bound_input_provider) { + handle = 0; + const bool enabled = RknnRgaBoundInputEnabled(); + const bool compatible = shared_resource && shared_resource->rknn_bound_input_preprocess_compatible; + const bool provider_available = shared_resource && shared_resource->rknn_bound_input_provider; + if (!detector_contract_ || rga_bound_target_unavailable_ || !enabled || !compatible || + !provider_available) { + if (detector_contract_ && !rga_bound_target_unavailable_ && !rga_bound_guard_logged_) { + LOG_WARN( + "RKNN detector cannot acquire the RGA-bound input target: enabled={} shared_resource={} " + "preprocess_compatible={} provider={}", + enabled, shared_resource != nullptr, compatible, provider_available); + rga_bound_guard_logged_ = true; + } return false; } std::string reason; @@ -319,7 +438,7 @@ bool RknnResizeNode::ResizeWithRga(const Blob& bottom, Blob& top, bool allow_bou const int source_width = bottom_desc.dims[2]; const auto source_size = PackedByteCount(source_width, source_height); const auto target_size = PackedByteCount(out_width_, out_height_); - ScopedRgaHandle host_target_handle; + media::ScopedRgaBufferHandle host_target_handle; uint32_t target_handle = 0; const bool bound_target = allow_bound_target && AcquireRgaBoundTarget(target_handle); if (!bound_target) { @@ -340,18 +459,25 @@ bool RknnResizeNode::ResizeWithRga(const Blob& bottom, Blob& top, bool allow_bou IM_STATUS last_status = IM_STATUS_FAILED; const auto run_resize = [&](rga_buffer_handle_t source_handle, int visible_width, int visible_height, - int width_stride, int height_stride, int source_format, bool yuv_source) { + int width_stride, int height_stride, int source_format, + IM_COLOR_SPACE_MODE yuv_color_space) { auto source = wrapbuffer_handle_t(source_handle, visible_width, visible_height, width_stride, height_stride, source_format); - if (yuv_source) - source.color_space_mode = IM_YUV_TO_RGB_BT601_LIMIT; + if (yuv_color_space != IM_COLOR_SPACE_DEFAULT) { + // Modern librga interprets color_space_mode as the color range of + // each buffer. Writing the legacy conversion selector (0x1) + // directly into the source descriptor is rejected by legacy + // RGA2-Pro runtimes. Describe source and destination ranges explicitly; + // improcess then performs CSC and resize in the same DMA-BUF job. + media::SetRgaYuvToRgbColorSpace(source, target, yuv_color_space); + } const auto fill_started = MetricsClock::now(); const im_rect full_target{0, 0, out_width_, out_height_}; const int fill_color = (padding_color_[0] << 16) | (padding_color_[1] << 8) | padding_color_[2]; last_status = imfill_t(target, full_target, fill_color, 1); GetInferencePipelineMetrics().RecordRknnRgaFill(ElapsedNanoseconds(fill_started)); - if (!RgaSucceeded(last_status)) + if (!media::RockchipRgaSucceeded(last_status)) return false; const float scale = std::min(static_cast(out_width_) / visible_width, @@ -367,7 +493,7 @@ bool RknnResizeNode::ResizeWithRga(const Blob& bottom, Blob& top, bool allow_bou const auto resize_started = MetricsClock::now(); last_status = improcess(source, target, empty_buffer, source_rect, target_rect, empty_rect, IM_SYNC); GetInferencePipelineMetrics().RecordRknnRgaResizeColor(ElapsedNanoseconds(resize_started)); - return RgaSucceeded(last_status); + return media::RockchipRgaSucceeded(last_status); }; const auto& native = bottom_handle.native_image; @@ -376,7 +502,7 @@ bool RknnResizeNode::ResizeWithRga(const Blob& bottom, Blob& top, bool allow_bou (native.format == IMAGE_NV12 || native.format == IMAGE_I420); if (native_compatible) { bool native_success = false; - ScopedRgaHandle native_source_handle; + media::ScopedRgaBufferHandle native_source_handle; if (!RknnForceMppDmaBufFailure()) { const auto import_started = MetricsClock::now(); native_source_handle.ImportFd(native.fd, native.bytes); @@ -385,8 +511,9 @@ bool RknnResizeNode::ResizeWithRga(const Blob& bottom, Blob& top, bool allow_bou if (native_source_handle.Get() != 0) { const int native_format = native.format == IMAGE_NV12 ? RK_FORMAT_YCbCr_420_SP : RK_FORMAT_YCbCr_420_P; - native_success = run_resize(native_source_handle.Get(), native.width, native.height, - native.width_stride, native.height_stride, native_format, true); + native_success = + run_resize(native_source_handle.Get(), native.width, native.height, native.width_stride, + native.height_stride, native_format, ResolveRgaYuvColorSpace(native)); } } if (native_success) { @@ -407,12 +534,12 @@ bool RknnResizeNode::ResizeWithRga(const Blob& bottom, Blob& top, bool allow_bou } } - ScopedRgaHandle host_source_handle(bottom_handle.base, source_size); + media::ScopedRgaBufferHandle host_source_handle(bottom_handle.base, source_size); const int host_source_format = bottom_desc.image_format == IMAGE_RGB ? RK_FORMAT_RGB_888 : RK_FORMAT_BGR_888; if (host_source_handle.Get() == 0 || !run_resize(host_source_handle.Get(), source_width, source_height, source_width, source_height, - host_source_format, false)) { + host_source_format, IM_COLOR_SPACE_DEFAULT)) { GetInferencePipelineMetrics().RecordRknnRgaFailure(); LogRgaFallbackOnce(host_source_handle.Get() == 0 ? IM_STATUS_OUT_OF_MEMORY : last_status); return false; @@ -530,6 +657,315 @@ Status RknnResizeNode::Forward(std::vector>& bottom_blobs, return COSMO_NN_OK; } +RknnCropResizeNode::RknnCropResizeNode() : CpuCropResizeNode() { + node_type = NodeType::NODE_CROP_RESIZE; + name = NodeTypeUtils::NodeTypeToStr(NODE_CROP_RESIZE).append("_0"); + one_blob_only = false; +} + +RknnCropResizeNode::~RknnCropResizeNode() { + ReleaseRgaBoundTarget(); +} + +void RknnCropResizeNode::LoadParam(Op* op) { + CpuCropResizeNode::LoadParam(op); + fast_contract_ = dst_height > 0 && dst_width > 0 && !h_top_crop.empty() && !h_bottom_crop.empty() && + !w_left_crop.empty() && !w_right_crop.empty(); +} + +void RknnCropResizeNode::ReleaseRgaBoundTarget() { + if (rga_bound_target_handle_ != 0) { + releasebuffer_handle(static_cast(rga_bound_target_handle_)); + rga_bound_target_handle_ = 0; + rga_bound_target_generation_ = 0; + } +} + +void RknnCropResizeNode::InvalidateRgaBoundFrame() { + if (!shared_resource) + return; + auto& target = shared_resource->rknn_bound_input_target; + if (target.owner == shared_resource->rknn_bound_input_provider) + target.frame_ready = false; +} + +bool RknnCropResizeNode::AcquireRgaBoundTarget(uint32_t& handle) { + handle = 0; + const bool enabled = RknnRgaBoundInputEnabled(); + const bool compatible = shared_resource && shared_resource->rknn_bound_input_preprocess_compatible; + const bool provider_available = shared_resource && shared_resource->rknn_bound_input_provider; + if (!enabled || !compatible || !provider_available || rga_bound_target_unavailable_) { + if (!rga_bound_target_unavailable_ && !rga_bound_guard_logged_) { + LOG_WARN( + "RKNN classifier cannot acquire the RGA-bound input target: enabled={} " + "shared_resource={} preprocess_compatible={} provider={}", + enabled, shared_resource != nullptr, compatible, provider_available); + rga_bound_guard_logged_ = true; + } + return false; + } + std::string reason; + auto* provider = shared_resource->rknn_bound_input_provider; + if (!provider->EnsureRgaBoundInput(dst_height, dst_width, reason)) { + rga_bound_target_unavailable_ = true; + LogRgaBoundInputFallbackOnce(reason); + return false; + } + const auto& target = shared_resource->rknn_bound_input_target; + if (target.owner != provider || !target.Matches(dst_height, dst_width) || + target.bytes > static_cast(std::numeric_limits::max())) { + rga_bound_target_unavailable_ = true; + LogRgaBoundInputFallbackOnce("provider returned an incompatible crop-resize target"); + return false; + } + if (rga_bound_target_handle_ != 0 && rga_bound_target_generation_ == target.generation) { + handle = rga_bound_target_handle_; + return true; + } + ReleaseRgaBoundTarget(); + const auto import_started = MetricsClock::now(); + const auto imported = importbuffer_fd(target.fd, static_cast(target.bytes)); + GetInferencePipelineMetrics().RecordRknnRgaBoundInputImport(ElapsedNanoseconds(import_started), + imported != 0); + if (imported == 0) { + rga_bound_target_unavailable_ = true; + GetInferencePipelineMetrics().RecordRknnRgaFailure(); + LogRgaBoundInputFallbackOnce("RGA could not import the classifier RKNN DMA-BUF fd"); + return false; + } + rga_bound_target_handle_ = static_cast(imported); + rga_bound_target_generation_ = target.generation; + handle = rga_bound_target_handle_; + return true; +} + +bool RknnCropResizeNode::ForwardWithRga(std::vector>& image_blobs, + std::vector>& rect_blobs, + std::vector>& top_blobs) { + if (!fast_contract_ || RknnForceRgaFailure() || image_blobs.empty() || + image_blobs.size() != rect_blobs.size() || top_blobs.size() != 1 || !top_blobs[0] || + !top_blobs[0]->GetHandle().base) { + if (fast_contract_ && RknnForceRgaFailure()) { + GetInferencePipelineMetrics().RecordRknnRgaCropResize(0, false); + GetInferencePipelineMetrics().RecordRknnRgaFailure(); + } + return false; + } + + int current_batch = 0; + for (const auto& rect_blob : rect_blobs) { + if (!rect_blob || rect_blob->GetBlobDesc().dims.size() != 2) + return false; + current_batch += rect_blob->GetBlobDesc().dims[0]; + } + if (current_batch <= 0 || current_batch > max_batch) + return false; + + auto top_desc = top_blobs[0]->GetBlobDesc(); + top_desc.dims[0] = current_batch; + top_blobs[0]->SetBlobDesc(top_desc); + auto* top_data = static_cast(top_blobs[0]->GetHandle().base); + const size_t slice_size = PackedByteCount(dst_width, dst_height); + int processed = 0; + bool bound_target = false; + uint32_t bound_handle = 0; + if (current_batch == 1) + bound_target = AcquireRgaBoundTarget(bound_handle); + + IM_STATUS last_status = IM_STATUS_FAILED; + for (size_t image_index = 0; image_index < image_blobs.size(); ++image_index) { + const auto& image_blob = image_blobs[image_index]; + if (!image_blob || !image_blob->GetHandle().base) + return false; + const auto image_desc = image_blob->GetBlobDesc(); + const auto image_handle = image_blob->GetHandle(); + if (image_desc.data_type != DATA_TYPE_UINT8 || image_desc.data_format != DATA_FORMAT_NHWC || + image_desc.dims.size() != 4 || image_desc.dims[0] != 1 || image_desc.dims[1] <= 0 || + image_desc.dims[2] <= 0 || image_desc.dims[3] != 3 || + (image_desc.image_format != IMAGE_BGR && image_desc.image_format != IMAGE_RGB)) { + return false; + } + const int source_height = image_desc.dims[1]; + const int source_width = image_desc.dims[2]; + auto rect_status = PrepareRect(rect_blobs[image_index], source_width, source_height); + if (!rect_status) + return false; + + const auto& native = image_handle.native_image; + const bool native_compatible = RknnMppDmaBufEnabled() && native.Valid() && + native.width == source_width && native.height == source_height && + (native.format == IMAGE_NV12 || native.format == IMAGE_I420); + media::ScopedRgaBufferHandle native_source_handle; + bool native_source_ready = false; + int native_source_format = RK_FORMAT_UNKNOWN; + if (native_compatible && !RknnForceMppDmaBufFailure()) { + const auto import_started = MetricsClock::now(); + native_source_handle.ImportFd(native.fd, native.bytes); + native_source_ready = bool(native_source_handle); + GetInferencePipelineMetrics().RecordRknnMppDmaBufImport(ElapsedNanoseconds(import_started), + native_source_ready); + native_source_format = + native.format == IMAGE_NV12 ? RK_FORMAT_YCbCr_420_SP : RK_FORMAT_YCbCr_420_P; + } + if (native_compatible && !native_source_ready) { + GetInferencePipelineMetrics().RecordRknnMppDmaBufFallback(); + LogMppDmaBufFallbackOnce(IM_STATUS_OUT_OF_MEMORY); + } + + media::ScopedRgaBufferHandle host_source_handle; + const int host_source_format = + image_desc.image_format == IMAGE_RGB ? RK_FORMAT_RGB_888 : RK_FORMAT_BGR_888; + + const int rect_count = rect_blobs[image_index]->GetBlobDesc().dims[0]; + for (int rect_index = 0; rect_index < rect_count; ++rect_index) { + const int crop_x = calculated_rects[4 * rect_index]; + const int crop_y = calculated_rects[4 * rect_index + 1]; + const int crop_width = calculated_rects[4 * rect_index + 2]; + const int crop_height = calculated_rects[4 * rect_index + 3]; + if (crop_width <= 0 || crop_height <= 0) + return false; + + media::ScopedRgaBufferHandle host_target_handle; + uint32_t target_handle = bound_target ? bound_handle : 0; + int target_stride = dst_width; + if (bound_target) { + target_stride = shared_resource->rknn_bound_input_target.width_stride; + } else { + host_target_handle.ImportVirtual(top_data + static_cast(processed) * slice_size, + slice_size); + target_handle = host_target_handle.Get(); + } + if (target_handle == 0) { + last_status = IM_STATUS_OUT_OF_MEMORY; + GetInferencePipelineMetrics().RecordRknnRgaCropResize(0, false); + GetInferencePipelineMetrics().RecordRknnRgaFailure(); + InvalidateRgaBoundFrame(); + LogRgaCropResizeFallbackOnce(last_status); + return false; + } + + auto target = wrapbuffer_handle_t(static_cast(target_handle), dst_width, + dst_height, target_stride, dst_height, RK_FORMAT_RGB_888); + int resized_width = dst_width; + int resized_height = dst_height; + int offset_x = 0; + int offset_y = 0; + if (gravity != 0) { + const float scale = std::min(static_cast(dst_width) / crop_width, + static_cast(dst_height) / crop_height); + resized_width = static_cast(crop_width * scale); + resized_height = static_cast(crop_height * scale); + if (gravity == 1) { + offset_x = (dst_width - resized_width) / 2; + offset_y = (dst_height - resized_height) / 2; + } + const uint8_t padding = static_cast(color.empty() ? 114 : color[0]); + const int fill_color = (padding << 16) | (padding << 8) | padding; + const im_rect full_target{0, 0, dst_width, dst_height}; + last_status = imfill_t(target, full_target, fill_color, 1); + if (!media::RockchipRgaSucceeded(last_status)) { + GetInferencePipelineMetrics().RecordRknnRgaCropResize(0, false); + GetInferencePipelineMetrics().RecordRknnRgaFailure(); + InvalidateRgaBoundFrame(); + LogRgaCropResizeFallbackOnce(last_status); + return false; + } + } + if (resized_width <= 0 || resized_height <= 0) + return false; + + const im_rect target_rect{offset_x, offset_y, resized_width, resized_height}; + const auto run_crop = [&](rga_buffer_t source, const im_rect& source_rect, + IM_COLOR_SPACE_MODE source_color_space) { + return RunStagedRgaCropResize(source, source_rect, source_color_space, target, target_rect, + last_status); + }; + + bool success = false; + if (native_source_ready) { + auto native_source = + wrapbuffer_handle_t(native_source_handle.Get(), source_width, source_height, + native.width_stride, native.height_stride, native_source_format); + const auto native_rect = + AlignYuv420Crop(crop_x, crop_y, crop_width, crop_height, source_width, source_height); + success = native_rect.width > 0 && native_rect.height > 0 && + run_crop(native_source, native_rect, ResolveRgaYuvColorSpace(native)); + if (success) { + GetInferencePipelineMetrics().RecordRknnMppDmaBufFrame(native.bytes); + GetInferencePipelineMetrics().RecordRknnRgaCropSource(true); + } else { + native_source_ready = false; + GetInferencePipelineMetrics().RecordRknnMppDmaBufFallback(); + GetInferencePipelineMetrics().RecordRknnRgaFailure(); + LogMppDmaBufFallbackOnce(last_status); + } + } + + if (!success) { + if (!host_source_handle) { + host_source_handle.ImportVirtual(image_handle.base, + PackedByteCount(source_width, source_height)); + } + if (host_source_handle) { + auto host_source = + wrapbuffer_handle_t(host_source_handle.Get(), source_width, source_height, + source_width, source_height, host_source_format); + const im_rect source_rect{crop_x, crop_y, crop_width, crop_height}; + success = run_crop(host_source, source_rect, IM_COLOR_SPACE_DEFAULT); + if (success) + GetInferencePipelineMetrics().RecordRknnRgaCropSource(false); + } else { + last_status = IM_STATUS_OUT_OF_MEMORY; + GetInferencePipelineMetrics().RecordRknnRgaCropResize(0, false); + } + } + if (!success) { + GetInferencePipelineMetrics().RecordRknnRgaFailure(); + InvalidateRgaBoundFrame(); + LogRgaCropResizeFallbackOnce(last_status); + return false; + } + GetInferencePipelineMetrics().RecordRknnPreprocessFastHit(); + ++processed; + } + } + + if (bound_target) { + auto& target = shared_resource->rknn_bound_input_target; + if (target.owner == shared_resource->rknn_bound_input_provider && + target.generation == rga_bound_target_generation_) { + target.frame_ready = true; + } + } + top_desc.data_format = DATA_FORMAT_NHWC; + top_desc.image_format = IMAGE_RGB; + top_blobs[0]->SetBlobDesc(top_desc); + return processed == current_batch; +} + +Status RknnCropResizeNode::Forward(std::vector>& image_blobs, + std::vector>& rect_blobs, + std::vector>& top_blobs) { + InvalidateRgaBoundFrame(); + timer.Start(); + if (ForwardWithRga(image_blobs, rect_blobs, top_blobs)) { + timer.Stop(); + return COSMO_NN_OK; + } + InvalidateRgaBoundFrame(); + timer.Stop(); + const auto fallback_started = MetricsClock::now(); + auto status = CpuCropResizeNode::Forward(image_blobs, rect_blobs, top_blobs); + GetInferencePipelineMetrics().RecordRknnCpuCropResizeFallback(ElapsedNanoseconds(fallback_started)); + if (status && !top_blobs.empty() && top_blobs[0] && !image_blobs.empty() && image_blobs[0]) { + auto fallback_desc = top_blobs[0]->GetBlobDesc(); + fallback_desc.data_format = DATA_FORMAT_NHWC; + fallback_desc.image_format = image_blobs[0]->GetBlobDesc().image_format; + top_blobs[0]->SetBlobDesc(fallback_desc); + } + return status; +} + RknnNormalizeNode::RknnNormalizeNode() : Node() { node_type = NodeType::NODE_NORMALIZE; name = NodeTypeUtils::NodeTypeToStr(NODE_NORMALIZE).append("_0"); @@ -568,6 +1004,8 @@ Status RknnNormalizeNode::InferTopShapesWithBottoms(std::vector dims detector_sized_ = dims[0][1] == kDetectorInputSize && dims[0][2] == kDetectorInputSize && dims[0][3] == 3; native_contract_ = types[0] == DATA_TYPE_UINT8 && IsRknnNativeNormalizeContract(mean_, std_dev_, uniform_scale_, dims[0]); + if (shared_resource) + shared_resource->rknn_bound_input_preprocess_compatible = native_contract_ && !is_bgr_; if (native_contract_) { top_blob_shapes = {dims[0]}; top_blob_data_types = {DataType::DATA_TYPE_INT8}; @@ -686,6 +1124,10 @@ Status RknnNormalizeNode::Forward(std::vector>& bottom_blo GetInferencePipelineMetrics().RecordRknnRgaBoundInputNormalizeBypass(); status = COSMO_NN_OK; } else { + if (shared_resource && shared_resource->rknn_bound_input_target.owner == + shared_resource->rknn_bound_input_provider) { + shared_resource->rknn_bound_input_target.frame_ready = false; + } status = ForwardNative(*bottom_blobs[0], *top_blobs[0]); } } else { diff --git a/src/nn/device/rknn/rknn_preprocess_node.h b/src/nn/device/rknn/rknn_preprocess_node.h index 47ba89852..dcc19f6b1 100644 --- a/src/nn/device/rknn/rknn_preprocess_node.h +++ b/src/nn/device/rknn/rknn_preprocess_node.h @@ -6,6 +6,7 @@ #include #include +#include "nn/device/cpu/cpu_crop_resize_node.h" #include "nn/node/node.h" namespace cosmo::nn { @@ -49,6 +50,32 @@ class RknnResizeNode final : public Node { uint32_t rga_bound_target_handle_{0}; uint64_t rga_bound_target_generation_{0}; bool rga_bound_target_unavailable_{false}; + bool rga_bound_guard_logged_{false}; +}; + +class RknnCropResizeNode final : public CpuCropResizeNode { +public: + RknnCropResizeNode(); + ~RknnCropResizeNode() override; + + void LoadParam(Op* op) override; + Status Forward(std::vector>& image_blobs, + std::vector>& rect_blobs, + std::vector>& top_blobs) override; + +private: + bool ForwardWithRga(std::vector>& image_blobs, + std::vector>& rect_blobs, + std::vector>& top_blobs); + bool AcquireRgaBoundTarget(uint32_t& handle); + void ReleaseRgaBoundTarget(); + void InvalidateRgaBoundFrame(); + + bool fast_contract_{false}; + uint32_t rga_bound_target_handle_{0}; + uint64_t rga_bound_target_generation_{0}; + bool rga_bound_target_unavailable_{false}; + bool rga_bound_guard_logged_{false}; }; class RknnNormalizeNode final : public Node { diff --git a/src/nn/pipeline/classify_pipeline.cc b/src/nn/pipeline/classify_pipeline.cc index 8cba0c8bf..4697ebf89 100644 --- a/src/nn/pipeline/classify_pipeline.cc +++ b/src/nn/pipeline/classify_pipeline.cc @@ -18,11 +18,12 @@ Status ClassifyPipeline::Init(const PipelineConfig& config, const std::string& m nlohmann::json p = pipeline_utils::ParseJsonObject(mc.params_json); ModelInfo model; - model.name = mc.name; - model.filename = mc.file_name; - model.file_md5 = mc.file_md5; - model.max_batch = mc.max_batch; - max_batch_ = mc.max_batch; + model.name = mc.name; + model.filename = mc.file_name; + model.file_md5 = mc.file_md5; + model.max_batch = mc.max_batch; + max_batch_ = mc.max_batch; + model.input_contract = pipeline_utils::ReadString(p, "rknn_input_contract", std::string()); std::vector dsize = pipeline_utils::ReadIntArray(p, "input_size", {224, 224}, 2); diff --git a/src/service/model/impl/ModelAddModel.cc b/src/service/model/impl/ModelAddModel.cc index 189592d6e..f2df68cf1 100644 --- a/src/service/model/impl/ModelAddModel.cc +++ b/src/service/model/impl/ModelAddModel.cc @@ -180,14 +180,18 @@ util::ErrorEnum ModelImportExporter::ValidateAddModelInputs( namespace fs = std::filesystem; #ifdef COSMO_NN_USE_RKNN_BACKEND - static const std::vector kSupportedRknnModelTypes = { - // Expose only model types verified end-to-end on a real RK3576 device. - // Keypoints, feature extraction, OCR and Grounding DINO remain disabled - // until their matching RKNN models pass runtime validation. - "yolov8_det", "classify", "qwen3_5"}; + static const std::vector kSupportedRknnModelTypes = [] { + // CV types share the RKNN tensor-contract path. Optional model families + // are exposed by compiled capability, not by a hard-coded chip name. + std::vector types{"yolov8_det", "classify"}; +#ifdef COSMO_NN_USE_RKLLM_BACKEND + types.emplace_back("qwen3_5"); +#endif + return types; + }(); if (std::find(kSupportedRknnModelTypes.begin(), kSupportedRknnModelTypes.end(), modelType) == kSupportedRknnModelTypes.end()) { - LOG_WARN("[AddModel] Unsupported RK3576 model type: {}", modelType); + LOG_WARN("[AddModel] Unsupported {} model type: {}", cosmo::util::kEngineType, modelType); return util::ErrorEnum::InvalidParam; } #endif @@ -216,10 +220,10 @@ util::ErrorEnum ModelImportExporter::ValidateAddModelInputs( } bool is_sam2 = (modelType == "sam2"); -#ifdef COSMO_NN_USE_RKNN_BACKEND +#ifdef COSMO_NN_USE_RKLLM_BACKEND const bool is_rkllm_qwen35 = (modelType == "qwen3_5"); if (is_rkllm_qwen35 && bmodel_files.size() != 2) { - LOG_WARN("{}", "[AddModel] RK3576 Qwen3.5 requires model.rkllm and vision.rknn"); + LOG_WARN("[AddModel] {} Qwen3.5 requires model.rkllm and vision.rknn", cosmo::util::kEngineType); return util::ErrorEnum::InvalidParam; } #else @@ -268,7 +272,7 @@ util::ErrorEnum ModelImportExporter::ValidateAddModelInputs( // Check all model files exist. Keep RKLLM files in a deterministic language/vision order. std::vector ordered_model_files; ordered_model_files.reserve(bmodel_files.size()); -#ifdef COSMO_NN_USE_RKNN_BACKEND +#ifdef COSMO_NN_USE_RKLLM_BACKEND if (is_rkllm_qwen35) { const cosmo::Model::BmodelFileInfo* language_model = nullptr; const cosmo::Model::BmodelFileInfo* vision_model = nullptr; @@ -279,7 +283,8 @@ util::ErrorEnum ModelImportExporter::ValidateAddModelInputs( vision_model = &file; } if (!language_model || !vision_model) { - LOG_WARN("{}", "[AddModel] RK3576 Qwen3.5 model roles must be language and vision"); + LOG_WARN("[AddModel] {} Qwen3.5 model roles must be language and vision", + cosmo::util::kEngineType); return util::ErrorEnum::InvalidParam; } // Staged upload paths are opaque and intentionally do not preserve client filenames. @@ -375,9 +380,12 @@ util::ErrorEnum ModelImportExporter::WriteNnFile(const std::string& modelType, #ifdef COSMO_NN_USE_RKNN_BACKEND std::string convert_error; +#ifdef COSMO_NN_USE_RKLLM_BACKEND + const bool is_rkllm_qwen35 = (modelType == "qwen3_5"); if (modelType == "qwen3_5") { if (bmodel_paths.size() != 2) { - convert_error = "RK3576 Qwen3.5 requires one RKLLM file and one vision RKNN file"; + convert_error = std::string(cosmo::util::kEngineType) + + " Qwen3.5 requires one RKLLM file and one vision RKNN file"; } else { const std::vector destination_names = {"model.rkllm", "vision.rknn"}; for (size_t i = 0; i < destination_names.size(); ++i) { @@ -391,17 +399,21 @@ util::ErrorEnum ModelImportExporter::WriteNnFile(const std::string& modelType, LOG_INFO("[AddModel] RKLLM: copied component to {}", destination); } } - } else if (bmodel_paths.size() != 1) { - convert_error = "RKNN add-model requires exactly one model file"; - } else { - const std::string model_file_path = model_dir + "/model.rknn"; - std::error_code ec; - fs::copy_file(bmodel_paths[0], model_file_path, fs::copy_options::overwrite_existing, ec); - if (ec) - convert_error = "Failed to copy RKNN model to " + model_file_path + ": " + ec.message(); - else - LOG_INFO("[AddModel] RKNN: copied model file to {}", model_file_path); - } +#endif +#ifndef COSMO_NN_USE_RKLLM_BACKEND + const bool is_rkllm_qwen35 = false; +#endif + if (!is_rkllm_qwen35 && bmodel_paths.size() != 1) { + convert_error = "RKNN add-model requires exactly one model file"; + } else if (!is_rkllm_qwen35) { + const std::string model_file_path = model_dir + "/model.rknn"; + std::error_code ec; + fs::copy_file(bmodel_paths[0], model_file_path, fs::copy_options::overwrite_existing, ec); + if (ec) + convert_error = "Failed to copy RKNN model to " + model_file_path + ": " + ec.message(); + else + LOG_INFO("[AddModel] RKNN: copied model file to {}", model_file_path); + } #elif defined(COSMO_NN_USE_ONNX_BACKEND) // CPU/x86: copy .onnx file directly; no Sophon wrapper needed. std::string convert_error; @@ -443,264 +455,262 @@ util::ErrorEnum ModelImportExporter::WriteNnFile(const std::string& modelType, } #endif - if (!convert_error.empty()) { - try { - fs::remove_all(model_dir); - } catch (const std::exception&) { + if (!convert_error.empty()) { + try { + fs::remove_all(model_dir); + } catch (const std::exception&) { + } + LOG_WARN("[AddModel] model file conversion/copy failed: {}", convert_error); + return util::ErrorEnum::SysErr; } - LOG_WARN("[AddModel] model file conversion/copy failed: {}", convert_error); - return util::ErrorEnum::SysErr; + return util::ErrorEnum::Success; } - return util::ErrorEnum::Success; -} -// UpdateTemplateConfig moved to ModelAddModel_Json.cc + // UpdateTemplateConfig moved to ModelAddModel_Json.cc -util::ErrorEnum ModelImportExporter::CopyAuxiliaryFiles(const std::string& modelType, - const std::string& vocabFilePath, - const std::string& tokenizerFilePath, - const std::string& characterTableFilePath, - const std::string& model_dir) { - namespace fs = std::filesystem; + util::ErrorEnum ModelImportExporter::CopyAuxiliaryFiles( + const std::string& modelType, const std::string& vocabFilePath, const std::string& tokenizerFilePath, + const std::string& characterTableFilePath, const std::string& model_dir) { + namespace fs = std::filesystem; - if (modelType == "dino" && !vocabFilePath.empty()) { - if (fs::exists(vocabFilePath)) { - std::string dest_vocab = model_dir + "/vocab.txt"; - try { - fs::copy_file(vocabFilePath, dest_vocab, fs::copy_options::overwrite_existing); - LOG_INFO("[AddModel] Copied vocab.txt to {}", dest_vocab); - } catch (const std::exception& e) { + if (modelType == "dino" && !vocabFilePath.empty()) { + if (fs::exists(vocabFilePath)) { + std::string dest_vocab = model_dir + "/vocab.txt"; + try { + fs::copy_file(vocabFilePath, dest_vocab, fs::copy_options::overwrite_existing); + LOG_INFO("[AddModel] Copied vocab.txt to {}", dest_vocab); + } catch (const std::exception& e) { + try { + fs::remove_all(model_dir); + } catch (const std::exception&) { + } + LOG_WARN("[AddModel] Failed to copy vocab.txt: {}", e.what()); + return util::ErrorEnum::SysErr; + } + } else { try { fs::remove_all(model_dir); } catch (const std::exception&) { } - LOG_WARN("[AddModel] Failed to copy vocab.txt: {}", e.what()); - return util::ErrorEnum::SysErr; + LOG_WARN("[AddModel] vocab.txt temp file does not exist: {}", vocabFilePath); + return util::ErrorEnum::FileNotExist; } - } else { - try { - fs::remove_all(model_dir); - } catch (const std::exception&) { + } + + if ((modelType == "qwen3vl" || modelType == "qwen3_5") && !tokenizerFilePath.empty()) { + if (fs::exists(tokenizerFilePath)) { + std::string dest_tokenizer = model_dir + "/tokenizer.json"; + try { + fs::copy_file(tokenizerFilePath, dest_tokenizer, fs::copy_options::overwrite_existing); + LOG_INFO("[AddModel] Copied tokenizer.json to {}", dest_tokenizer); + } catch (const std::exception& e) { + try { + fs::remove_all(model_dir); + } catch (const std::exception&) { + } + LOG_WARN("[AddModel] Failed to copy tokenizer.json: {}", e.what()); + return util::ErrorEnum::SysErr; + } + } else { + try { + fs::remove_all(model_dir); + } catch (const std::exception&) { + } + LOG_WARN("[AddModel] tokenizer.json temp file does not exist: {}", tokenizerFilePath); + return util::ErrorEnum::FileNotExist; } - LOG_WARN("[AddModel] vocab.txt temp file does not exist: {}", vocabFilePath); - return util::ErrorEnum::FileNotExist; } - } - if ((modelType == "qwen3vl" || modelType == "qwen3_5") && !tokenizerFilePath.empty()) { - if (fs::exists(tokenizerFilePath)) { - std::string dest_tokenizer = model_dir + "/tokenizer.json"; + if (modelType == "ocr") { + std::string dest_table = model_dir + "/character_table.txt"; try { - fs::copy_file(tokenizerFilePath, dest_tokenizer, fs::copy_options::overwrite_existing); - LOG_INFO("[AddModel] Copied tokenizer.json to {}", dest_tokenizer); + fs::copy_file(characterTableFilePath, dest_table, fs::copy_options::overwrite_existing); + LOG_INFO("[AddModel] Copied OCR character table to {}", dest_table); } catch (const std::exception& e) { try { fs::remove_all(model_dir); } catch (const std::exception&) { } - LOG_WARN("[AddModel] Failed to copy tokenizer.json: {}", e.what()); + LOG_WARN("[AddModel] Failed to copy OCR character table: {}", e.what()); return util::ErrorEnum::SysErr; } - } else { - try { - fs::remove_all(model_dir); - } catch (const std::exception&) { - } - LOG_WARN("[AddModel] tokenizer.json temp file does not exist: {}", tokenizerFilePath); - return util::ErrorEnum::FileNotExist; } - } - if (modelType == "ocr") { - std::string dest_table = model_dir + "/character_table.txt"; - try { - fs::copy_file(characterTableFilePath, dest_table, fs::copy_options::overwrite_existing); - LOG_INFO("[AddModel] Copied OCR character table to {}", dest_table); - } catch (const std::exception& e) { - try { - fs::remove_all(model_dir); - } catch (const std::exception&) { - } - LOG_WARN("[AddModel] Failed to copy OCR character table: {}", e.what()); - return util::ErrorEnum::SysErr; - } + return util::ErrorEnum::Success; } - return util::ErrorEnum::Success; -} + // ============================================================ + // Slim AddAtomicModel orchestrator + // ============================================================ + util::ErrorEnum ModelImportExporter::AddAtomicModel( + const std::string& modelCode, const std::string& modelName, const std::string& modelType, + const std::string& description, const std::vector& bmodel_files, + const std::string& vocabFilePath, const std::string& tokenizerFilePath, + const std::string& characterTableFilePath, const std::string& normalizationMode, + const std::string& colorChannel) { + namespace fs = std::filesystem; -// ============================================================ -// Slim AddAtomicModel orchestrator -// ============================================================ -util::ErrorEnum ModelImportExporter::AddAtomicModel( - const std::string& modelCode, const std::string& modelName, const std::string& modelType, - const std::string& description, const std::vector& bmodel_files, - const std::string& vocabFilePath, const std::string& tokenizerFilePath, - const std::string& characterTableFilePath, const std::string& normalizationMode, - const std::string& colorChannel) { - namespace fs = std::filesystem; + const std::string models_dir = get_model_path_(); + const std::string template_dir = get_model_template_path_(); - const std::string models_dir = get_model_path_(); - const std::string template_dir = get_model_template_path_(); + // Collect temp files for cleanup + std::vector temp_files_to_cleanup; + for (const auto& bmodelFile : bmodel_files) { + if (!bmodelFile.filePath.empty()) + temp_files_to_cleanup.push_back(bmodelFile.filePath); + } + if (!vocabFilePath.empty()) + temp_files_to_cleanup.push_back(vocabFilePath); + if (!tokenizerFilePath.empty()) + temp_files_to_cleanup.push_back(tokenizerFilePath); + if (!characterTableFilePath.empty()) + temp_files_to_cleanup.push_back(characterTableFilePath); + + auto cleanup_and_return = [&](util::ErrorEnum err) -> util::ErrorEnum { + CleanupManagedUploadFiles(temp_files_to_cleanup); + return err; + }; - // Collect temp files for cleanup - std::vector temp_files_to_cleanup; - for (const auto& bmodelFile : bmodel_files) { - if (!bmodelFile.filePath.empty()) - temp_files_to_cleanup.push_back(bmodelFile.filePath); - } - if (!vocabFilePath.empty()) - temp_files_to_cleanup.push_back(vocabFilePath); - if (!tokenizerFilePath.empty()) - temp_files_to_cleanup.push_back(tokenizerFilePath); - if (!characterTableFilePath.empty()) - temp_files_to_cleanup.push_back(characterTableFilePath); - - auto cleanup_and_return = [&](util::ErrorEnum err) -> util::ErrorEnum { - CleanupManagedUploadFiles(temp_files_to_cleanup); - return err; - }; - - // 1-2. Validate inputs - std::string resolved_model_code; - std::vector bmodel_paths; - auto err = - ValidateAddModelInputs(modelCode, modelName, modelType, bmodel_files, vocabFilePath, - tokenizerFilePath, characterTableFilePath, resolved_model_code, bmodel_paths); - if (err != util::ErrorEnum::Success) - return cleanup_and_return(err); - - std::string resolved_vocab_path; - std::string resolved_tokenizer_path; - std::string resolved_character_table_path; - if ((!vocabFilePath.empty() && !ResolveManagedUploadFile(vocabFilePath, resolved_vocab_path)) || - (!tokenizerFilePath.empty() && - !ResolveManagedUploadFile(tokenizerFilePath, resolved_tokenizer_path)) || - (!characterTableFilePath.empty() && - !ResolveManagedUploadFile(characterTableFilePath, resolved_character_table_path))) { - LOG_WARN("{}", "[AddModel] Auxiliary file is not a managed upload"); - return cleanup_and_return(util::ErrorEnum::FileNotExist); - } + // 1-2. Validate inputs + std::string resolved_model_code; + std::vector bmodel_paths; + auto err = ValidateAddModelInputs(modelCode, modelName, modelType, bmodel_files, vocabFilePath, + tokenizerFilePath, characterTableFilePath, resolved_model_code, + bmodel_paths); + if (err != util::ErrorEnum::Success) + return cleanup_and_return(err); - // 3. Read template file - std::string template_path = (fs::path(template_dir) / (modelType + ".json")).string(); - if (!fs::exists(template_path)) { - LOG_WARN("[AddModel] Model type template not found: {}", template_path); - return cleanup_and_return(util::ErrorEnum::FileNotExist); - } + std::string resolved_vocab_path; + std::string resolved_tokenizer_path; + std::string resolved_character_table_path; + if ((!vocabFilePath.empty() && !ResolveManagedUploadFile(vocabFilePath, resolved_vocab_path)) || + (!tokenizerFilePath.empty() && + !ResolveManagedUploadFile(tokenizerFilePath, resolved_tokenizer_path)) || + (!characterTableFilePath.empty() && + !ResolveManagedUploadFile(characterTableFilePath, resolved_character_table_path))) { + LOG_WARN("{}", "[AddModel] Auxiliary file is not a managed upload"); + return cleanup_and_return(util::ErrorEnum::FileNotExist); + } - std::ifstream templateFile(template_path); - if (!templateFile.is_open()) { - LOG_WARN("[AddModel] Failed to open template file: {}", template_path); - return cleanup_and_return(util::ErrorEnum::SysErr); - } + // 3. Read template file + std::string template_path = (fs::path(template_dir) / (modelType + ".json")).string(); + if (!fs::exists(template_path)) { + LOG_WARN("[AddModel] Model type template not found: {}", template_path); + return cleanup_and_return(util::ErrorEnum::FileNotExist); + } - std::stringstream templateBuffer; - templateBuffer << templateFile.rdbuf(); - templateFile.close(); + std::ifstream templateFile(template_path); + if (!templateFile.is_open()) { + LOG_WARN("[AddModel] Failed to open template file: {}", template_path); + return cleanup_and_return(util::ErrorEnum::SysErr); + } - nlohmann::json templateDoc; - try { - templateDoc = nlohmann::json::parse(templateBuffer.str()); - } catch (const std::exception& e) { - LOG_WARN("[AddModel] Template file JSON parse error: {} ({})", template_path, e.what()); - return cleanup_and_return(util::ErrorEnum::InvalidParam); - } + std::stringstream templateBuffer; + templateBuffer << templateFile.rdbuf(); + templateFile.close(); - // 4. Get bmodel info - bool use_template_defaults = false; - std::vector bmodel_infos; - err = CollectBmodelInfo(modelType, bmodel_paths, bmodel_infos, use_template_defaults); - if (err != util::ErrorEnum::Success) - return cleanup_and_return(err); + nlohmann::json templateDoc; + try { + templateDoc = nlohmann::json::parse(templateBuffer.str()); + } catch (const std::exception& e) { + LOG_WARN("[AddModel] Template file JSON parse error: {} ({})", template_path, e.what()); + return cleanup_and_return(util::ErrorEnum::InvalidParam); + } - if (modelType == "ocr") { - err = ConfigureOcrCharacterTable(templateDoc, bmodel_infos, resolved_character_table_path); + // 4. Get bmodel info + bool use_template_defaults = false; + std::vector bmodel_infos; + err = CollectBmodelInfo(modelType, bmodel_paths, bmodel_infos, use_template_defaults); if (err != util::ErrorEnum::Success) return cleanup_and_return(err); - } - // 5. Calculate version number - std::string version_str = CalculateNextVersion(models_dir, resolved_model_code); - - // 6. Create model directory - std::string clean_model_name = modelName; - std::replace(clean_model_name.begin(), clean_model_name.end(), ' ', '_'); - std::replace(clean_model_name.begin(), clean_model_name.end(), '/', '_'); - std::replace(clean_model_name.begin(), clean_model_name.end(), '\\', '_'); - - std::string folder_name = std::string(cosmo::util::kNewDirPrefix) + resolved_model_code + "_" + - clean_model_name + "_" + version_str; - std::string model_dir; - if (!cosmo::path::IsSafePathComponent(folder_name, 200) || - !cosmo::path::ResolvePathWithinRoot(models_dir, (fs::path(models_dir) / folder_name).string(), - model_dir)) { - LOG_WARN("[AddModel] Refusing unsafe model directory component: {}", folder_name); - return cleanup_and_return(util::ErrorEnum::InvalidParam); - } + if (modelType == "ocr") { + err = ConfigureOcrCharacterTable(templateDoc, bmodel_infos, resolved_character_table_path); + if (err != util::ErrorEnum::Success) + return cleanup_and_return(err); + } - LOG_INFO("[AddModel] Creating model directory: {}", model_dir); - if (!util::CreateDir(model_dir)) { - LOG_WARN("[AddModel] Failed to create model directory: {}", model_dir); - return cleanup_and_return(util::ErrorEnum::SysErr); - } + // 5. Calculate version number + std::string version_str = CalculateNextVersion(models_dir, resolved_model_code); + + // 6. Create model directory + std::string clean_model_name = modelName; + std::replace(clean_model_name.begin(), clean_model_name.end(), ' ', '_'); + std::replace(clean_model_name.begin(), clean_model_name.end(), '/', '_'); + std::replace(clean_model_name.begin(), clean_model_name.end(), '\\', '_'); + + std::string folder_name = std::string(cosmo::util::kNewDirPrefix) + resolved_model_code + "_" + + clean_model_name + "_" + version_str; + std::string model_dir; + if (!cosmo::path::IsSafePathComponent(folder_name, 200) || + !cosmo::path::ResolvePathWithinRoot(models_dir, (fs::path(models_dir) / folder_name).string(), + model_dir)) { + LOG_WARN("[AddModel] Refusing unsafe model directory component: {}", folder_name); + return cleanup_and_return(util::ErrorEnum::InvalidParam); + } - // 7. Write model.nn - err = WriteNnFile(modelType, bmodel_paths, model_dir); - if (err != util::ErrorEnum::Success) - return cleanup_and_return(err); + LOG_INFO("[AddModel] Creating model directory: {}", model_dir); + if (!util::CreateDir(model_dir)) { + LOG_WARN("[AddModel] Failed to create model directory: {}", model_dir); + return cleanup_and_return(util::ErrorEnum::SysErr); + } - // 8. Update template config - UpdateTemplateConfig(templateDoc, resolved_model_code, version_str, modelName, modelType, description, - bmodel_infos, use_template_defaults, normalizationMode, colorChannel); + // 7. Write model.nn + err = WriteNnFile(modelType, bmodel_paths, model_dir); + if (err != util::ErrorEnum::Success) + return cleanup_and_return(err); - // 8.1 Validate model output format - { - try { - validate_model_output_format_(templateDoc); - } catch (const std::exception&) { + // 8. Update template config + UpdateTemplateConfig(templateDoc, resolved_model_code, version_str, modelName, modelType, description, + bmodel_infos, use_template_defaults, normalizationMode, colorChannel); + + // 8.1 Validate model output format + { try { - fs::remove_all(model_dir); + validate_model_output_format_(templateDoc); } catch (const std::exception&) { + try { + fs::remove_all(model_dir); + } catch (const std::exception&) { + } + CleanupManagedUploadFiles(temp_files_to_cleanup); + throw; // re-throw to preserve original exception behavior } - CleanupManagedUploadFiles(temp_files_to_cleanup); - throw; // re-throw to preserve original exception behavior } - } - // 9. Save config.json - std::string config_path = model_dir + "/config.json"; - std::string json_content = templateDoc.dump(2); + // 9. Save config.json + std::string config_path = model_dir + "/config.json"; + std::string json_content = templateDoc.dump(2); - if (!util::WriteFile(config_path, json_content)) { - try { - fs::remove_all(model_dir); - } catch (const std::exception&) { + if (!util::WriteFile(config_path, json_content)) { + try { + fs::remove_all(model_dir); + } catch (const std::exception&) { + } + LOG_WARN("[AddModel] Failed to save config.json: {}", config_path); + return cleanup_and_return(util::ErrorEnum::SysErr); } - LOG_WARN("[AddModel] Failed to save config.json: {}", config_path); - return cleanup_and_return(util::ErrorEnum::SysErr); - } - // 9.1 Copy auxiliary files - err = CopyAuxiliaryFiles(modelType, resolved_vocab_path, resolved_tokenizer_path, - resolved_character_table_path, model_dir); - if (err != util::ErrorEnum::Success) - return cleanup_and_return(err); + // 9.1 Copy auxiliary files + err = CopyAuxiliaryFiles(modelType, resolved_vocab_path, resolved_tokenizer_path, + resolved_character_table_path, model_dir); + if (err != util::ErrorEnum::Success) + return cleanup_and_return(err); - // 10. Clean up temp files - CleanupManagedUploadFiles(temp_files_to_cleanup); + // 10. Clean up temp files + CleanupManagedUploadFiles(temp_files_to_cleanup); - // 11. Update memory mapping so service can find it without restart - if (set_model_path_mapping_) { - set_model_path_mapping_(resolved_model_code, model_dir); - } + // 11. Update memory mapping so service can find it without restart + if (set_model_path_mapping_) { + set_model_path_mapping_(resolved_model_code, model_dir); + } - LOG_INFO("[AddModel] Successfully added model: code={}, name={}, version={}, directory={}", - resolved_model_code, modelName, version_str, model_dir); + LOG_INFO("[AddModel] Successfully added model: code={}, name={}, version={}, directory={}", + resolved_model_code, modelName, version_str, model_dir); - return util::ErrorEnum::Success; -} + return util::ErrorEnum::Success; + } -// ImportFlatArchive, ImportDirectoryArchive, ImportModel — moved to ModelImporter.cc + // ImportFlatArchive, ImportDirectoryArchive, ImportModel — moved to ModelImporter.cc } // namespace cosmo::service diff --git a/src/service/system/dto/SystemMsgTypes.cc b/src/service/system/dto/SystemMsgTypes.cc index 8c7cc9f9f..720a4e832 100644 --- a/src/service/system/dto/SystemMsgTypes.cc +++ b/src/service/system/dto/SystemMsgTypes.cc @@ -93,6 +93,12 @@ void from_json(const nlohmann::json& j, MsgGpuInfo& v) { JSON_OPT(j, v, mppCopyOutFrames); JSON_OPT(j, v, mppCopyOutMs); JSON_OPT(j, v, mppCopyOutFailures); + JSON_OPT(j, v, mppRgaCopyOutFrames); + JSON_OPT(j, v, mppRgaCopyOutFailures); + JSON_OPT(j, v, mppCpuCopyOutFallbacks); + JSON_OPT(j, v, mppRgaCopyInFrames); + JSON_OPT(j, v, mppRgaCopyInFailures); + JSON_OPT(j, v, mppCpuCopyInFallbacks); JSON_OPT(j, v, mppEarlyDroppedFrames); JSON_OPT(j, v, colorConvertFrames); JSON_OPT(j, v, colorConvertMs); @@ -143,9 +149,16 @@ void from_json(const nlohmann::json& j, MsgGpuInfo& v) { JSON_OPT(j, v, rknnRgaFillMs); JSON_OPT(j, v, rknnRgaResizeColorCalls); JSON_OPT(j, v, rknnRgaResizeColorMs); + JSON_OPT(j, v, rknnRgaCropResizeCalls); + JSON_OPT(j, v, rknnRgaCropResizeMs); + JSON_OPT(j, v, rknnRgaCropResizeFailures); + JSON_OPT(j, v, rknnRgaCropDmaBufFrames); + JSON_OPT(j, v, rknnRgaCropHostFallbacks); JSON_OPT(j, v, rknnRgaFailures); JSON_OPT(j, v, rknnCpuResizeFallbackCalls); JSON_OPT(j, v, rknnCpuResizeFallbackMs); + JSON_OPT(j, v, rknnCpuCropResizeFallbackCalls); + JSON_OPT(j, v, rknnCpuCropResizeFallbackMs); JSON_OPT(j, v, rknnCpuNormalizeFallbackCalls); JSON_OPT(j, v, rknnCpuNormalizeFallbackMs); JSON_OPT(j, v, rknnNativeInputMapCalls); @@ -249,6 +262,12 @@ void to_json(nlohmann::json& j, const MsgGpuInfo& v) { j["mppCopyOutFrames"] = v.mppCopyOutFrames; j["mppCopyOutMs"] = v.mppCopyOutMs; j["mppCopyOutFailures"] = v.mppCopyOutFailures; + j["mppRgaCopyOutFrames"] = v.mppRgaCopyOutFrames; + j["mppRgaCopyOutFailures"] = v.mppRgaCopyOutFailures; + j["mppCpuCopyOutFallbacks"] = v.mppCpuCopyOutFallbacks; + j["mppRgaCopyInFrames"] = v.mppRgaCopyInFrames; + j["mppRgaCopyInFailures"] = v.mppRgaCopyInFailures; + j["mppCpuCopyInFallbacks"] = v.mppCpuCopyInFallbacks; j["mppEarlyDroppedFrames"] = v.mppEarlyDroppedFrames; j["colorConvertFrames"] = v.colorConvertFrames; j["colorConvertMs"] = v.colorConvertMs; @@ -299,9 +318,16 @@ void to_json(nlohmann::json& j, const MsgGpuInfo& v) { j["rknnRgaFillMs"] = v.rknnRgaFillMs; j["rknnRgaResizeColorCalls"] = v.rknnRgaResizeColorCalls; j["rknnRgaResizeColorMs"] = v.rknnRgaResizeColorMs; + j["rknnRgaCropResizeCalls"] = v.rknnRgaCropResizeCalls; + j["rknnRgaCropResizeMs"] = v.rknnRgaCropResizeMs; + j["rknnRgaCropResizeFailures"] = v.rknnRgaCropResizeFailures; + j["rknnRgaCropDmaBufFrames"] = v.rknnRgaCropDmaBufFrames; + j["rknnRgaCropHostFallbacks"] = v.rknnRgaCropHostFallbacks; j["rknnRgaFailures"] = v.rknnRgaFailures; j["rknnCpuResizeFallbackCalls"] = v.rknnCpuResizeFallbackCalls; j["rknnCpuResizeFallbackMs"] = v.rknnCpuResizeFallbackMs; + j["rknnCpuCropResizeFallbackCalls"] = v.rknnCpuCropResizeFallbackCalls; + j["rknnCpuCropResizeFallbackMs"] = v.rknnCpuCropResizeFallbackMs; j["rknnCpuNormalizeFallbackCalls"] = v.rknnCpuNormalizeFallbackCalls; j["rknnCpuNormalizeFallbackMs"] = v.rknnCpuNormalizeFallbackMs; j["rknnNativeInputMapCalls"] = v.rknnNativeInputMapCalls; diff --git a/src/service/system/dto/SystemMsgTypes.h b/src/service/system/dto/SystemMsgTypes.h index 7a66362cf..92342128a 100644 --- a/src/service/system/dto/SystemMsgTypes.h +++ b/src/service/system/dto/SystemMsgTypes.h @@ -78,6 +78,12 @@ struct MsgGpuInfo { uint64_t mppCopyOutFrames{0}; double mppCopyOutMs{0.0}; uint64_t mppCopyOutFailures{0}; + uint64_t mppRgaCopyOutFrames{0}; + uint64_t mppRgaCopyOutFailures{0}; + uint64_t mppCpuCopyOutFallbacks{0}; + uint64_t mppRgaCopyInFrames{0}; + uint64_t mppRgaCopyInFailures{0}; + uint64_t mppCpuCopyInFallbacks{0}; uint64_t mppEarlyDroppedFrames{0}; uint64_t colorConvertFrames{0}; double colorConvertMs{0.0}; @@ -128,9 +134,16 @@ struct MsgGpuInfo { double rknnRgaFillMs{0.0}; uint64_t rknnRgaResizeColorCalls{0}; double rknnRgaResizeColorMs{0.0}; + uint64_t rknnRgaCropResizeCalls{0}; + double rknnRgaCropResizeMs{0.0}; + uint64_t rknnRgaCropResizeFailures{0}; + uint64_t rknnRgaCropDmaBufFrames{0}; + uint64_t rknnRgaCropHostFallbacks{0}; uint64_t rknnRgaFailures{0}; uint64_t rknnCpuResizeFallbackCalls{0}; double rknnCpuResizeFallbackMs{0.0}; + uint64_t rknnCpuCropResizeFallbackCalls{0}; + double rknnCpuCropResizeFallbackMs{0.0}; uint64_t rknnCpuNormalizeFallbackCalls{0}; double rknnCpuNormalizeFallbackMs{0.0}; uint64_t rknnNativeInputMapCalls{0}; diff --git a/src/service/system/impl/AcceleratorMetricsProviderRknn.cc b/src/service/system/impl/AcceleratorMetricsProviderRknn.cc index 4ad99d65f..6b94c4743 100644 --- a/src/service/system/impl/AcceleratorMetricsProviderRknn.cc +++ b/src/service/system/impl/AcceleratorMetricsProviderRknn.cc @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -14,6 +16,7 @@ #include #include "service/system/impl/AcceleratorMetricsProvider.h" +#include "util/NnBackendConstants.h" namespace cosmo::service::detail { namespace { @@ -105,13 +108,47 @@ namespace { return result; } - std::string ReadNpuFrequency() { - std::ifstream stream("/sys/class/devfreq/27700000.npu/cur_freq"); + std::optional ReadFrequencyFile(const std::filesystem::path& path) { + std::ifstream stream(path); uint64_t hz = 0; - if (!(stream >> hz)) - return "RK3576 shared-memory NPU"; + if (!(stream >> hz) || hz == 0) + return std::nullopt; + return hz; + } + + std::string ReadNpuFrequency() { + std::vector candidates; + if (const char* configured_path = std::getenv("COSMO_RKNPU_FREQ_PATH"); + configured_path && *configured_path) { + candidates.emplace_back(configured_path); + } + + std::error_code error; + const std::filesystem::path devfreq_root("/sys/class/devfreq"); + for (std::filesystem::directory_iterator it(devfreq_root, error), end; !error && it != end; + it.increment(error)) { + std::string identity = it->path().filename().string(); + std::ifstream name_stream(it->path() / "name"); + identity.append(std::istreambuf_iterator(name_stream), std::istreambuf_iterator()); + std::transform(identity.begin(), identity.end(), identity.begin(), + [](unsigned char value) { return static_cast(std::tolower(value)); }); + if (identity.find("npu") != std::string::npos) + candidates.push_back(it->path() / "cur_freq"); + } + std::sort(candidates.begin(), candidates.end()); + candidates.erase(std::unique(candidates.begin(), candidates.end()), candidates.end()); + + for (const auto& path : candidates) { + const auto hz = ReadFrequencyFile(path); + if (!hz) + continue; + std::ostringstream text; + text << (*hz / 1000000) << " MHz"; + return text.str(); + } + std::ostringstream text; - text << (hz / 1000000) << " MHz"; + text << cosmo::util::kEngineType << " shared-memory NPU"; return text.str(); } diff --git a/src/service/system/impl/DeviceInfoServiceImpl.cc b/src/service/system/impl/DeviceInfoServiceImpl.cc index 5442d8a98..660f388c3 100644 --- a/src/service/system/impl/DeviceInfoServiceImpl.cc +++ b/src/service/system/impl/DeviceInfoServiceImpl.cc @@ -243,7 +243,7 @@ std::vector DeviceInfoServiceImpl::GetHardwareResource(double& c FormatBinaryMebibytes(usage.available), usage.valid ? 1 : 0, memory_domain}); }; - // RK3576 has no dedicated NPU VRAM: its legacy gpumem fields describe the + // Rockchip RKNN devices use shared system memory: legacy gpumem fields describe the // same system DDR already represented above. Keep those fields for model // admission and wire compatibility, but never emit a second UI capacity. if (gpu_utl.memoryDomain != "shared-system") { diff --git a/src/service/system/impl/HardwareQueryUtil.cc b/src/service/system/impl/HardwareQueryUtil.cc index 373eb5100..a2c0f8966 100644 --- a/src/service/system/impl/HardwareQueryUtil.cc +++ b/src/service/system/impl/HardwareQueryUtil.cc @@ -30,6 +30,7 @@ #include "util/Exec.h" #include "util/FileUtil.h" #include "util/Log.h" +#include "util/NnBackendConstants.h" #include "util/PathUtil.h" #include "util/TimingConstants.h" @@ -107,7 +108,7 @@ void HardwareQueryUtil::ReadDeviceSnAndModel(std::string* device_sn, std::string *device_sn = QueryPrimaryMac(); *device_model = ReadDeviceTreeText("/proc/device-tree/model"); if (device_model->empty()) - *device_model = "RK3576"; + *device_model = cosmo::util::kEngineType; LOG_INFO("RKNN deviceSn:{} deviceModel:{}", *device_sn, *device_model); return; #elif !defined(COSMO_NN_USE_SOPHON_BACKEND) @@ -148,7 +149,7 @@ void HardwareQueryUtil::ReadDeviceSnAndModel(std::string* device_sn, std::string std::string HardwareQueryUtil::ReadHardwareSpec() { #if defined(COSMO_NN_USE_RKNN_BACKEND) auto compatible = ReadDeviceTreeText("/proc/device-tree/compatible", ','); - return compatible.empty() ? "rockchip,rk3576" : compatible; + return compatible.empty() ? cosmo::util::kHardwareSpecFallback : compatible; #elif !defined(COSMO_NN_USE_SOPHON_BACKEND) return "X86_HARDWARE_SPEC"; #else @@ -169,7 +170,7 @@ std::string HardwareQueryUtil::ReadKernelRevision() { struct utsname system_info {}; if (uname(&system_info) == 0) return system_info.release; - LOG_WARN("uname failed while querying RK3576 kernel revision: {}", std::strerror(errno)); + LOG_WARN("uname failed while querying RKNN kernel revision: {}", std::strerror(errno)); return {}; #elif !defined(COSMO_NN_USE_SOPHON_BACKEND) return "X86-Generic-Kernel"; diff --git a/src/util/NnBackendConstants.h b/src/util/NnBackendConstants.h index e4dc5de47..12f96f303 100644 --- a/src/util/NnBackendConstants.h +++ b/src/util/NnBackendConstants.h @@ -38,7 +38,9 @@ static constexpr const char* kNewDirPrefix = "prod_SOPHGO_"; static constexpr const char* kPlatformDirRegex = "prod_[A-Z0-9]+_([0-9]+)_.*"; /// Engine type identifier reported to frontend / device info API. -static constexpr const char* kEngineType = "BM1688"; +static constexpr const char* kBackendType = "SOPHON"; +static constexpr const char* kEngineType = "BM1688"; +static constexpr bool kSupportsRkllm = false; /// Model binary file extension for Sophon backend (.nn wraps .bmodel). static constexpr const char* kModelFileExt = ".nn"; @@ -49,13 +51,29 @@ static constexpr const char* kSupportedChips[] = {"BM1688", "CV186X"}; #elif defined(COSMO_NN_USE_RKNN_BACKEND) -/// Directory prefix for RK3576 RKNN model directories. -static constexpr const char* kPlatformDirPrefix = "prod_RK3576_"; -static constexpr const char* kNewDirPrefix = "prod_RK3576_"; -static constexpr const char* kPlatformDirRegex = "prod_[A-Z0-9]+_([0-9]+)_.*"; -static constexpr const char* kEngineType = "RK3576"; -static constexpr const char* kModelFileExt = ".rknn"; -static constexpr const char* kSupportedChips[] = {"RK3576"}; +#ifndef COSMO_RKNN_TARGET_CHIP +#error "RKNN builds must define COSMO_RKNN_TARGET_CHIP through COSMO_TARGET_CHIP" +#endif +#ifndef COSMO_RKNN_TARGET_CHIP_LABEL +#error "RKNN builds must define COSMO_RKNN_TARGET_CHIP_LABEL through COSMO_TARGET_CHIP" +#endif + +/// Legacy target-labelled directories remain readable. Newly imported models +/// use a vendor-level token; config.json chip_type is the compatibility gate. +static constexpr const char* kPlatformDirPrefix = "prod_" COSMO_RKNN_TARGET_CHIP_LABEL "_"; +static constexpr const char* kNewDirPrefix = "prod_ROCKCHIP_"; +static constexpr const char* kPlatformDirRegex = "prod_[A-Z0-9]+_([0-9]+)_.*"; +static constexpr const char* kBackendType = "RKNN"; +static constexpr const char* kEngineType = COSMO_RKNN_TARGET_CHIP_LABEL; +static constexpr const char* kTargetChip = COSMO_RKNN_TARGET_CHIP; +static constexpr const char* kHardwareSpecFallback = "rockchip," COSMO_RKNN_TARGET_CHIP; +#ifdef COSMO_NN_USE_RKLLM_BACKEND +static constexpr bool kSupportsRkllm = true; +#else +static constexpr bool kSupportsRkllm = false; +#endif +static constexpr const char* kModelFileExt = ".rknn"; +static constexpr const char* kSupportedChips[] = {COSMO_RKNN_TARGET_CHIP_LABEL}; #elif defined(COSMO_NN_USE_CPU_BACKEND) @@ -70,7 +88,9 @@ static constexpr const char* kNewDirPrefix = "prod_X86_"; static constexpr const char* kPlatformDirRegex = "prod_[A-Z0-9]+_([0-9]+)_.*"; /// Engine type identifier reported to frontend / device info API. -static constexpr const char* kEngineType = "X86"; +static constexpr const char* kBackendType = "ONNX_RUNTIME"; +static constexpr const char* kEngineType = "X86"; +static constexpr bool kSupportsRkllm = false; /// Model binary file extension for CPU backend (.onnx used directly). static constexpr const char* kModelFileExt = ".onnx"; diff --git a/src/web/src/views/gam/countManagement/algorithmicManagement/algorithmicStatus.vue b/src/web/src/views/gam/countManagement/algorithmicManagement/algorithmicStatus.vue index dd5687715..c93f5c79f 100644 --- a/src/web/src/views/gam/countManagement/algorithmicManagement/algorithmicStatus.vue +++ b/src/web/src/views/gam/countManagement/algorithmicManagement/algorithmicStatus.vue @@ -24,30 +24,17 @@ {{ item.modelName }} - - + - - - - - - - - - - - - diff --git a/src/web/src/views/gam/countManagement/atomicModel/index.vue b/src/web/src/views/gam/countManagement/atomicModel/index.vue index 75bb3f992..7def8d64f 100644 --- a/src/web/src/views/gam/countManagement/atomicModel/index.vue +++ b/src/web/src/views/gam/countManagement/atomicModel/index.vue @@ -117,11 +117,11 @@ - + {{ t('action.browse') }} @@ -582,9 +582,10 @@ const topBarData = computed(() => ({ })) const isX86 = ref(false) const isRknn = ref(false) +const isRkllm = ref(false) const modelFileExtension = computed(() => isRknn.value ? '.rknn' : (isX86.value ? '.onnx' : '.bmodel')) const addModelPrimaryFileExtension = computed(() => - isRknn.value && addModelForm.modelType === 'qwen3_5' + isRkllm.value && addModelForm.modelType === 'qwen3_5' ? '.rkllm' : modelFileExtension.value ) @@ -674,11 +675,8 @@ const modelTypeGroups = computed(() => { } ] if (!isRknn.value) return groups - const supported = new Set([ - // RK3576 capabilities are exposed only after end-to-end validation with a - // real model on the device. Keep code-only integrations hidden for now. - 'yolov8_det', 'classify', 'qwen3_5' - ]) + const supported = new Set(['yolov8_det', 'classify']) + if (isRkllm.value) supported.add('qwen3_5') return groups .map(group => ({ ...group, children: group.children.filter(item => supported.has(item.value)) })) .filter(group => group.children.length > 0) @@ -829,12 +827,12 @@ const addModelRules = { visionFile: [ { validator: (rule, value, callback) => { - if (!isRknn.value || addModelForm.modelType !== 'qwen3_5') { + if (!isRkllm.value || addModelForm.modelType !== 'qwen3_5') { callback() return } if (!addModelForm.visionFileList || addModelForm.visionFileList.length === 0) { - callback(new Error('RK3576 Qwen3.5 需要上传 vision.rknn')) + callback(new Error('RKNN Qwen3.5 需要上传 vision.rknn')) } else { callback() } @@ -1409,7 +1407,7 @@ const sureAddModel = async () => { proxy.$message.warning(t('validate.decoderFormatError', { ext })) return } - } else if (isRknn.value && addModelForm.modelType === 'qwen3_5') { + } else if (isRkllm.value && addModelForm.modelType === 'qwen3_5') { if ( !addModelForm.modelFileList || addModelForm.modelFileList.length === 0 @@ -1421,7 +1419,7 @@ const sureAddModel = async () => { !addModelForm.visionFileList || addModelForm.visionFileList.length === 0 ) { - proxy.$message.warning('RK3576 Qwen3.5 需要上传 vision.rknn') + proxy.$message.warning('RKNN Qwen3.5 需要上传 vision.rknn') return } const languageFile = @@ -1436,7 +1434,7 @@ const sureAddModel = async () => { proxy.$message.warning(t('validate.uploadFormatError', { ext: '.rknn' })) return } - } else if (isRknn.value && addModelForm.modelType === 'qwen3_5') { + } else if (isRkllm.value && addModelForm.modelType === 'qwen3_5') { const languageFile = addModelForm.modelFileList[0].raw || addModelForm.modelFileList[0] const visionFile = @@ -1598,11 +1596,17 @@ onMounted(() => { proxy.$API.queryDeviceInfo().then(res => { const devInfoList = res?.resData?.devInfoList || [] const deviceTypeItem = devInfoList.find(item => item.key === 'deviceType') - if (deviceTypeItem) { - const deviceType = deviceTypeItem.value.toLowerCase() - isX86.value = deviceType.includes('x86') - isRknn.value = deviceType.includes('rk3576') || deviceType.includes('rockchip') - } + const acceleratorBackendItem = devInfoList.find(item => item.key === 'acceleratorBackend') + const rkllmAvailableItem = devInfoList.find(item => item.key === 'rkllmAvailable') + const acceleratorBackend = acceleratorBackendItem?.value?.toLowerCase() || '' + const deviceType = deviceTypeItem?.value?.toLowerCase() || '' + isX86.value = acceleratorBackend === 'onnx_runtime' || deviceType.includes('x86') + // Prefer the backend capability reported by new engines. Device-name + // fallbacks keep compatibility with older releases without adding a + // list of Rockchip chip names to the frontend. + isRknn.value = acceleratorBackend === 'rknn' || + deviceType.includes('rk3576') || deviceType.includes('rockchip') + isRkllm.value = rkllmAvailableItem?.value?.toLowerCase() === 'true' }).catch(() => {}) } loadModelTypes() diff --git a/test/agent/test_agent_workflow.py b/test/agent/test_agent_workflow.py index 1a960806d..64aede703 100644 --- a/test/agent/test_agent_workflow.py +++ b/test/agent/test_agent_workflow.py @@ -262,6 +262,19 @@ def test_unspecified_toolchain_selects_capability_probe_without_version_pin(self self.assertIsNone(invalid) self.assertIn("must be an object", error) + def test_rknn_target_selects_rknn_toolkit2_package_and_module(self): + specification, error = agent_workflow._toolchain_spec( + {"targetChip": "rv1126b"} + ) + self.assertEqual(error, "") + self.assertEqual(specification["family"], "rknn") + self.assertEqual(specification["package"], "rknn-toolkit2") + self.assertEqual(specification["module"], "rknn.api") + self.assertEqual( + specification["officialReference"], + agent_workflow.RKNN_TOOLKIT2_OFFICIAL_REFERENCE, + ) + def test_target_chip_can_be_inferred_without_limiting_user_to_example_chips(self): contract = self._model_contract("other-chip") contract["parameters"].pop("targetChip") @@ -331,6 +344,37 @@ def test_windows_assessment_routes_to_linux_without_calling_windows_unsupported( details = " ".join(item["detail"] for item in report["routeCandidates"]) self.assertIn("不等于 CosmoEdge 不支持 Windows", details) + def test_rv1126b_assessment_routes_to_remote_rknn_toolkit2(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + contract = self._model_contract("rv1126b-assessment") + contract["parameters"]["targetChip"] = "rv1126b" + contract["parameters"]["developmentEnvironment"] = { + "os": "linux", + "architecture": "x86_64", + "reference": "isolated development host", + } + contract["authority"]["grants"] = ["remote-execution", "model-transfer"] + run_dir = root / "output" / "agent-runs" / contract["runId"] + run_dir.mkdir(parents=True) + (run_dir / "model.onnx").write_bytes(b"fixture") + contract_path = run_dir / "task-contract.json" + contract_path.write_text(json.dumps(contract), encoding="utf-8") + inventory = self._model_inventory() + inventory["host"].update({"os": "Darwin", "architecture": "arm64"}) + with mock.patch.object(agent_workflow, "host_inventory", return_value=inventory): + report = agent_workflow.assess_task_report( + contract_path, run_dir, contract, project_root=root + ) + self.assertEqual(report["routeVerdict"], "READY") + self.assertEqual( + report["recommendedRoute"], "remote-linux-rknn-toolkit2" + ) + self.assertEqual( + report["routeCandidates"][-1]["officialReference"], + agent_workflow.RKNN_TOOLKIT2_OFFICIAL_REFERENCE, + ) + def test_remote_linux_assessment_consolidates_missing_authority(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -405,6 +449,29 @@ def test_start_creates_private_agent_owned_run_from_ordinary_intent(self): self.assertEqual(stat.S_IMODE(copied.stat().st_mode), 0o600) self.assertTrue((run_dir / "route-assessment.json").is_file()) + def test_start_selects_single_model_when_other_materials_are_present(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + model = root / "detector.onnx" + video = root / "sample.mp4" + board = root / "board.pdf" + model.write_bytes(b"model") + video.write_bytes(b"video") + board.write_bytes(b"pdf") + with mock.patch.object( + agent_workflow, "host_inventory", return_value=self._model_inventory() + ): + _, _, contract, report = agent_workflow.create_task_run( + task="model-conversion", + objective="Convert the detector for RV1126B.", + materials=[model, video, board], + target_chip="rv1126b", + run_id="mixed-material-start", + project_root=root, + ) + self.assertEqual(contract["parameters"]["sourceModel"], "inputs/detector.onnx") + self.assertEqual(report["routeVerdict"], "READY") + def test_start_refuses_overwrite_and_pt_cannot_bypass_route_gate(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/test/agent/test_model_conversion_workflow.py b/test/agent/test_model_conversion_workflow.py index c030a64fe..0fbf6270f 100644 --- a/test/agent/test_model_conversion_workflow.py +++ b/test/agent/test_model_conversion_workflow.py @@ -6,6 +6,7 @@ import unittest from contextlib import redirect_stdout from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parents[2] @@ -135,6 +136,33 @@ def test_example_catalog_starts_truthfully_empty_and_json_assets_parse(self): ) self.assertIn("deviceValidation", schema["properties"]) + def test_checker_only_cli_does_not_request_runtime_execution(self): + with tempfile.TemporaryDirectory() as directory: + model_path = Path(directory) / "source.onnx" + report_path = Path(directory) / "report.json" + model_path.write_bytes(b"fixture") + result = { + "schemaVersion": "1.0", + "status": "PASS", + "validationMode": "checker-only", + "model": {"path": model_path.name, "sha256": "fixture"}, + "inputs": [], + "outputs": [], + } + with mock.patch.object( + check_onnx_model, "inspect_model", return_value=result + ) as inspect: + self.assertEqual( + check_onnx_model.main( + [str(model_path), "--checker-only", "--json", str(report_path)] + ), + 0, + ) + inspect.assert_called_once_with(model_path.resolve(), {}, run_runtime=False) + self.assertEqual( + json.loads(report_path.read_text())["validationMode"], "checker-only" + ) + def test_candidate_shapes_and_chip_come_from_contract(self): with tempfile.TemporaryDirectory() as directory: run_dir, _ = prepare_run(Path(directory), make_contract()) diff --git a/test/agent/test_rknn_agent_conversion_workflow.py b/test/agent/test_rknn_agent_conversion_workflow.py new file mode 100644 index 000000000..22e9bff9d --- /dev/null +++ b/test/agent/test_rknn_agent_conversion_workflow.py @@ -0,0 +1,291 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +import agent_workflow as core # noqa: E402 +import conversion_workflow_dispatch as dispatch # noqa: E402 +from rknn import agent_conversion_workflow as conversion # noqa: E402 +from rknn import stage_platform_resources as staging # noqa: E402 + + +def make_contract(run_id: str = "rknn-conversion-test") -> dict: + return { + "schemaVersion": "1.0", + "runId": run_id, + "task": "model-conversion", + "userObjective": "Convert the canonical YOLOv8 model for RV1126B.", + "expectedDeliverables": ["RV1126B RKNN model", "evidence"], + "allowedChanges": ["current run directory"], + "requiredCapabilities": [], + "acceptance": {}, + "authority": {"workspace": "isolated fixture"}, + "parameters": { + "sourceModel": "inputs/model.onnx", + "modelName": "yolov8", + "modelFamily": "YOLOv8 detector", + "targetBackend": "Rockchip RKNN", + "targetChip": "rv1126b", + "toolchainChip": "rv1126b", + "quantization": "INT8", + "inputLayout": "NCHW", + "inputShapes": [[1, 3, 640, 640]], + "expectedOutputShapes": [[1, 84, 8400]], + "pixelFormat": "rgb", + "outputKind": "rknn", + "toolchain": { + "kind": "auto", + "package": "rknn-toolkit2", + "module": "rknn.api", + "version": "2.3.2", + }, + "preflight": {"pythonExecutable": sys.executable, "pythonPackages": {}}, + }, + } + + +class RknnAgentConversionWorkflowTest(unittest.TestCase): + def test_source_and_normalized_input_use_distinct_onnx_checks(self): + source = Path("source.onnx") + report = Path("report.json") + checker = conversion._onnx_check_command( + sys.executable, source, report, checker_only=True + ) + runtime = conversion._onnx_check_command( + sys.executable, source, report, checker_only=False + ) + self.assertIn("--checker-only", checker) + self.assertNotIn("--checker-only", runtime) + self.assertEqual(checker[-2:], ["--json", str(report)]) + self.assertEqual(runtime[-2:], ["--json", str(report)]) + + def test_rv1126b_uses_shared_rknn_workflow_and_profile(self): + with tempfile.TemporaryDirectory() as directory: + run_dir = Path(directory) + inputs = run_dir / "inputs" + inputs.mkdir() + (inputs / "model.onnx").write_bytes(b"fixture") + video = inputs / "sample.mp4" + video.write_bytes(b"fixture-video") + parameters = conversion.conversion_parameters(make_contract(), run_dir) + self.assertEqual(parameters["profile"]["chip"], "rv1126b") + self.assertEqual(parameters["profile"]["backend"], "rknn") + self.assertEqual(parameters["toolchainLock"]["version"], "2.3.2") + self.assertEqual(parameters["calibrationSource"], video.resolve()) + + def test_model_spec_and_platform_profile_are_independent(self): + contract = make_contract() + with tempfile.TemporaryDirectory() as directory: + run_dir = Path(directory) + (run_dir / "inputs").mkdir() + (run_dir / "inputs" / "model.onnx").write_bytes(b"fixture") + (run_dir / "inputs" / "sample.mp4").write_bytes(b"fixture") + rv = conversion.conversion_parameters(contract, run_dir) + contract["parameters"]["targetChip"] = "rk3576" + contract["parameters"]["toolchainChip"] = "rk3576" + rk = conversion.conversion_parameters(contract, run_dir) + self.assertEqual(rv["specPath"], rk["specPath"]) + self.assertNotEqual(rv["profilePath"], rk["profilePath"]) + self.assertEqual(rv["toolchainLockPath"], rk["toolchainLockPath"]) + + def test_classifier_calibration_declares_shared_person_detector_contract(self): + contract = make_contract() + contract["parameters"]["modelName"] = "helmet" + contract["parameters"]["modelFamily"] = "helmet classifier" + contract["parameters"]["inputShapes"] = [[1, 3, 224, 224]] + contract["parameters"]["expectedOutputShapes"] = [[1, 2]] + with tempfile.TemporaryDirectory() as directory: + run_dir = Path(directory) + (run_dir / "inputs").mkdir() + (run_dir / "inputs" / "model.onnx").write_bytes(b"fixture") + (run_dir / "inputs" / "sample.mp4").write_bytes(b"fixture") + parameters = conversion.conversion_parameters(contract, run_dir) + self.assertEqual(parameters["personDetectorSpec"]["name"], "yolov8") + self.assertEqual( + parameters["personDetectorSpecPath"], + ROOT / "config" / "rknn" / "models" / "yolov8.json", + ) + self.assertEqual( + parameters["personDetector"], + ROOT + / "data" + / "resource" + / "aiboxresource_x86" + / "models" + / "prod_X86_9275710_YOLOV8_V1.0.0" + / "model.onnx", + ) + + def test_dispatch_selects_backend_family_from_contract(self): + rknn_contract = make_contract() + sophon_contract = make_contract() + sophon_contract["parameters"]["targetChip"] = "bm1688" + with mock.patch.object( + core, + "resolve_contract_context", + return_value=(Path("task-contract.json"), Path("run"), rknn_contract), + ): + self.assertEqual(dispatch.workflow_family(["--contract", "fixture"]), "rknn") + with mock.patch.object( + core, + "resolve_contract_context", + return_value=(Path("task-contract.json"), Path("run"), sophon_contract), + ): + self.assertEqual(dispatch.workflow_family(["--contract=fixture"]), "sophon") + + def test_repository_config_cannot_escape_backend_config_root(self): + contract = make_contract() + contract["parameters"]["modelSpec"] = "CMakeLists.txt" + with tempfile.TemporaryDirectory() as directory: + run_dir = Path(directory) + (run_dir / "inputs").mkdir() + (run_dir / "inputs" / "model.onnx").write_bytes(b"fixture") + (run_dir / "inputs" / "sample.mp4").write_bytes(b"fixture") + with self.assertRaisesRegex(core.WorkflowError, "must stay under"): + conversion.conversion_parameters(contract, run_dir) + + def test_resource_staging_rewrites_only_platform_identity(self): + source = { + "chip_type": "RK3576", + "algorithmName": "RK3576 Helmet", + "algorithmProcessdata": '[{"position":"rk3576-det","atomicCode":"9275710"}]', + "algorithmCode": 7463001, + } + staged = staging.replace_platform_tokens(source, "RK3576", "RV1126B") + self.assertEqual(staged["chip_type"], "RV1126B") + self.assertEqual(staged["algorithmName"], "RV1126B Helmet") + self.assertIn("rv1126b-det", staged["algorithmProcessdata"]) + self.assertEqual(staged["algorithmCode"], 7463001) + + def test_resource_staging_reads_embedded_atomic_contract(self): + document = { + "atomicList": json.dumps( + [ + {"atomicCode": "9275710"}, + {"atomicCode": "7982161"}, + ] + ) + } + self.assertEqual( + staging.algorithm_atomic_codes(document), {"9275710", "7982161"} + ) + + def _resource_staging_fixture(self, root: Path) -> tuple[Path, Path, Path]: + profile_path = root / "config/rknn/platforms/rv1126b.json" + profile_path.parent.mkdir(parents=True) + profile_path.write_text( + json.dumps( + { + "chip": "rv1126b", + "backend": "rknn", + "packaging": { + "directory_token": "RV1126B", + "resource_template_directory": "data/resource/template", + "resource_overlay_directory": ( + "output/platform-artifacts/rv1126b/resource-overlay" + ), + "resource_manifest_required": True, + }, + } + ), + encoding="utf-8", + ) + spec_path = root / "config/rknn/models/yolov8.json" + spec_path.parent.mkdir(parents=True) + spec_path.write_text( + json.dumps({"packaging": {"algorithm_code": "9275710"}}), + encoding="utf-8", + ) + template_config = ( + root + / "data/resource/template/models/" + "prod_RK3576_9275710_YOLOV8_V1.0.0/config.json" + ) + template_config.parent.mkdir(parents=True) + template_config.write_text( + json.dumps( + { + "algorithm_code": "9275710", + "chip_type": "RK3576", + "models": [ + { + "params": { + "rknn_input_contract": "cosmo.rknn.input.rgb_u8.v1" + } + } + ], + } + ), + encoding="utf-8", + ) + algorithm_path = root / "data/resource/template/algorithm/fixture_RK3576.json" + algorithm_path.parent.mkdir(parents=True) + algorithm_path.write_text( + json.dumps({"atomicList": [{"atomicCode": "9275710"}]}), + encoding="utf-8", + ) + artifact_path = root / "artifact.rknn" + artifact_path.write_bytes(b"rknn-fixture") + return profile_path, template_config, artifact_path + + def test_staged_resource_manifest_proves_current_source_and_artifact(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + profile_path, _, artifact_path = self._resource_staging_fixture(root) + with mock.patch.object(staging, "PROJECT_ROOT", root): + manifest = staging.stage_platform_resources( + profile_path, + [("yolov8", artifact_path)], + ) + result = staging.verify_staged_resources(profile_path) + self.assertEqual(manifest["schema_version"], 2) + self.assertEqual(result["status"], "PASS") + self.assertEqual(result["models"], ["yolov8"]) + source_record = manifest["models"][0]["source_template"] + self.assertTrue(source_record["path"].endswith("config.json")) + self.assertEqual(len(source_record["sha256"]), 64) + + def test_staged_resource_verification_rejects_changed_source_config(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + profile_path, template_config, artifact_path = ( + self._resource_staging_fixture(root) + ) + with mock.patch.object(staging, "PROJECT_ROOT", root): + staging.stage_platform_resources( + profile_path, + [("yolov8", artifact_path)], + ) + config = json.loads(template_config.read_text(encoding="utf-8")) + config["models"][0]["params"]["rknn_input_contract"] = "changed" + template_config.write_text(json.dumps(config), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "hash mismatch"): + staging.verify_staged_resources(profile_path) + + def test_staged_resource_verification_rejects_tampered_artifact(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + profile_path, _, artifact_path = self._resource_staging_fixture(root) + with mock.patch.object(staging, "PROJECT_ROOT", root): + manifest = staging.stage_platform_resources( + profile_path, + [("yolov8", artifact_path)], + ) + staged_artifact = ( + root + / "output/platform-artifacts/rv1126b/resource-overlay" + / manifest["models"][0]["artifact"]["path"] + ) + staged_artifact.write_bytes(b"tampered") + with self.assertRaisesRegex(ValueError, "hash mismatch"): + staging.verify_staged_resources(profile_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/agent/test_rknn_media_sysroot_lock.py b/test/agent/test_rknn_media_sysroot_lock.py new file mode 100644 index 000000000..42c0003c2 --- /dev/null +++ b/test/agent/test_rknn_media_sysroot_lock.py @@ -0,0 +1,175 @@ +import hashlib +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "media_sysroot_lock", ROOT / "tools" / "rknn" / "media_sysroot_lock.py" +) +assert SPEC and SPEC.loader +media_lock = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(media_lock) + + +def digest(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +class RknnMediaSysrootLockTest(unittest.TestCase): + def make_fixture( + self, directory: str, *, require_seal: bool = True + ) -> tuple[Path, Path]: + root = Path(directory) + sysroot = root / "sysroot" + (sysroot / "include" / "rockchip").mkdir(parents=True) + (sysroot / "include" / "rga").mkdir(parents=True) + (sysroot / "lib").mkdir(parents=True) + values = { + "include/rockchip/rk_mpi.h": b"mpp-header", + "include/rga/im2d.h": b"rga-header", + "lib/librockchip_mpp.so.0": b"mpp-library", + "lib/librga.so": b"rga-library", + } + for relative, value in values.items(): + (sysroot / relative).write_bytes(value) + (sysroot / "lib" / "librockchip_mpp.so.1").symlink_to( + "librockchip_mpp.so.0" + ) + (sysroot / "lib" / "librockchip_mpp.so").symlink_to( + "librockchip_mpp.so.1" + ) + + config = root / "config" + profile_dir = config / "rknn" / "platforms" + lock_dir = config / "rockchip-media" + profile_dir.mkdir(parents=True) + lock_dir.mkdir(parents=True) + profile = profile_dir / "fixture.json" + profile.write_text( + json.dumps( + { + "chip": "fixture", + "media": { + "default_backend": "rockchip", + "runtime_lock": "../../rockchip-media/runtime-lock.json", + "runtime_profile": "shared", + "require_sealed_sysroot": require_seal, + }, + } + ), + encoding="utf-8", + ) + (lock_dir / "runtime-lock.json").write_text( + json.dumps( + { + "runtimes": { + "shared": { + "sources": { + "mpp": {"revision": "mpp-revision"}, + "rga": {"revision": "rga-revision"}, + }, + "artifacts": { + relative: { + "sha256": digest(value), + **( + { + "elf": { + "machine": "AArch64", + "soname": ( + "librockchip_mpp.so.1" + if "mpp.so" in relative + else "librga.so" + ), + } + } + if relative.startswith("lib/") + else {} + ), + } + for relative, value in values.items() + }, + "links": { + "lib/librockchip_mpp.so": "librockchip_mpp.so.1", + "lib/librockchip_mpp.so.1": "librockchip_mpp.so.0", + }, + } + } + } + ), + encoding="utf-8", + ) + return profile, sysroot + + @mock.patch.object( + media_lock, + "inspect_elf", + side_effect=lambda path: { + "machine": "AArch64", + "soname": ( + "librockchip_mpp.so.1" + if "rockchip_mpp" in path.name + else "librga.so" + ), + }, + ) + def test_seal_and_verify_shared_runtime(self, _inspect: mock.Mock) -> None: + with tempfile.TemporaryDirectory() as directory: + profile, sysroot = self.make_fixture(directory) + sources = {"mpp": "mpp-revision", "rga": "rga-revision"} + manifest = media_lock.seal_sysroot(profile, sysroot, sources) + self.assertEqual(manifest.name, media_lock.MANIFEST_NAME) + result = media_lock.verify_sysroot(profile, sysroot) + self.assertEqual(result["runtime_profile"], "shared") + self.assertEqual(result["manifest"], manifest) + + (sysroot / "lib" / "librga.so").write_bytes(b"changed") + with self.assertRaisesRegex( + media_lock.MediaSysrootError, "hash mismatch" + ): + media_lock.verify_sysroot(profile, sysroot) + + @mock.patch.object( + media_lock, + "inspect_elf", + return_value={"machine": "AArch64", "soname": "librockchip_mpp.so.1"}, + ) + def test_seal_rejects_source_mismatch(self, _inspect: mock.Mock) -> None: + with tempfile.TemporaryDirectory() as directory: + profile, sysroot = self.make_fixture(directory) + with self.assertRaisesRegex( + media_lock.MediaSysrootError, "source revisions" + ): + media_lock.seal_sysroot( + profile, + sysroot, + {"mpp": "wrong", "rga": "rga-revision"}, + ) + + @mock.patch.object( + media_lock, + "inspect_elf", + side_effect=lambda path: { + "machine": "AArch64", + "soname": ( + "librockchip_mpp.so.1" + if "rockchip_mpp" in path.name + else "librga.so" + ), + }, + ) + def test_unsealed_legacy_runtime_remains_admissible( + self, _inspect: mock.Mock + ) -> None: + with tempfile.TemporaryDirectory() as directory: + profile, sysroot = self.make_fixture(directory, require_seal=False) + result = media_lock.verify_sysroot(profile, sysroot) + self.assertIsNone(result["manifest"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_inference_pipeline_metrics.cc b/test/test_inference_pipeline_metrics.cc index fdf36be86..6a4a51c65 100644 --- a/test/test_inference_pipeline_metrics.cc +++ b/test/test_inference_pipeline_metrics.cc @@ -23,8 +23,13 @@ TEST_CASE("Inference pipeline metrics expose host, graph, and RKNN stage timings metrics.RecordRknnPreprocessFastHit(); metrics.RecordRknnRgaFill(700'000); metrics.RecordRknnRgaResizeColor(800'000); + metrics.RecordRknnRgaCropResize(850'000, true); + metrics.RecordRknnRgaCropResize(50'000, false); + metrics.RecordRknnRgaCropSource(true); + metrics.RecordRknnRgaCropSource(false); metrics.RecordRknnRgaFailure(); metrics.RecordRknnCpuResizeFallback(900'000); + metrics.RecordRknnCpuCropResizeFallback(950'000); metrics.RecordRknnCpuNormalizeFallback(1'000'000); metrics.RecordRknnNativeInputMap(1'100'000); metrics.RecordRknnInputFormat(true); @@ -92,8 +97,15 @@ TEST_CASE("Inference pipeline metrics expose host, graph, and RKNN stage timings CHECK(snapshot.rknn_preprocess_fast_hits == 1); CHECK(snapshot.rknn_rga_fill_calls == 1); CHECK(snapshot.rknn_rga_resize_color_calls == 1); + CHECK(snapshot.rknn_rga_crop_resize_calls == 2); + CHECK(snapshot.rknn_rga_crop_resize_nanoseconds == 900'000); + CHECK(snapshot.rknn_rga_crop_resize_failures == 1); + CHECK(snapshot.rknn_rga_crop_dmabuf_frames == 1); + CHECK(snapshot.rknn_rga_crop_host_fallbacks == 1); CHECK(snapshot.rknn_rga_failures == 1); CHECK(snapshot.rknn_cpu_resize_fallback_calls == 1); + CHECK(snapshot.rknn_cpu_crop_resize_fallback_calls == 1); + CHECK(snapshot.rknn_cpu_crop_resize_fallback_nanoseconds == 950'000); CHECK(snapshot.rknn_cpu_normalize_fallback_calls == 1); CHECK(snapshot.rknn_native_input_map_calls == 1); CHECK(snapshot.rknn_native_int8_inputs == 1); diff --git a/test/test_json_serialization.cc b/test/test_json_serialization.cc index ecf404f76..32bc0fc8f 100644 --- a/test/test_json_serialization.cc +++ b/test/test_json_serialization.cc @@ -154,6 +154,17 @@ TEST_CASE("accelerator telemetry exposes additive load semantics", "[json][syste original.utilizationMetric = "busy-time-load"; original.coreUtilizations = {0.09, 0.42}; original.memoryDomain = "shared-system"; + original.mppRgaCopyOutFrames = 4; + original.mppRgaCopyInFrames = 3; + original.mppCpuCopyOutFallbacks = 1; + original.mppCpuCopyInFallbacks = 2; + original.rknnRgaCropResizeCalls = 7; + original.rknnRgaCropResizeMs = 3.5; + original.rknnRgaCropResizeFailures = 1; + original.rknnRgaCropDmaBufFrames = 6; + original.rknnRgaCropHostFallbacks = 1; + original.rknnCpuCropResizeFallbackCalls = 2; + original.rknnCpuCropResizeFallbackMs = 4.5; original.rknnYolov8DirectCandidateCalls = 4; original.rknnYolov8DirectCandidateFailures = 1; original.rknnYolov8DirectPointsScanned = 33'600; @@ -179,6 +190,12 @@ TEST_CASE("accelerator telemetry exposes additive load semantics", "[json][syste const auto doc = ParseJson(json); CHECK(doc["utilizationMetric"] == "busy-time-load"); CHECK(doc["memoryDomain"] == "shared-system"); + CHECK(doc["mppRgaCopyOutFrames"] == 4); + CHECK(doc["mppRgaCopyInFrames"] == 3); + CHECK(doc["rknnRgaCropResizeCalls"] == 7); + CHECK(doc["rknnRgaCropDmaBufFrames"] == 6); + CHECK(doc["rknnRgaCropHostFallbacks"] == 1); + CHECK(doc["rknnCpuCropResizeFallbackCalls"] == 2); REQUIRE(doc["coreUtilizations"].size() == 2); CHECK(doc["coreUtilizations"][0].get() == Catch::Approx(0.09)); CHECK(doc["coreUtilizations"][1].get() == Catch::Approx(0.42)); @@ -200,6 +217,17 @@ TEST_CASE("accelerator telemetry exposes additive load semantics", "[json][syste CHECK(restored.utilizationMetric == original.utilizationMetric); CHECK(restored.coreUtilizations == original.coreUtilizations); CHECK(restored.memoryDomain == original.memoryDomain); + CHECK(restored.mppRgaCopyOutFrames == original.mppRgaCopyOutFrames); + CHECK(restored.mppRgaCopyInFrames == original.mppRgaCopyInFrames); + CHECK(restored.mppCpuCopyOutFallbacks == original.mppCpuCopyOutFallbacks); + CHECK(restored.mppCpuCopyInFallbacks == original.mppCpuCopyInFallbacks); + CHECK(restored.rknnRgaCropResizeCalls == original.rknnRgaCropResizeCalls); + CHECK(restored.rknnRgaCropResizeMs == original.rknnRgaCropResizeMs); + CHECK(restored.rknnRgaCropResizeFailures == original.rknnRgaCropResizeFailures); + CHECK(restored.rknnRgaCropDmaBufFrames == original.rknnRgaCropDmaBufFrames); + CHECK(restored.rknnRgaCropHostFallbacks == original.rknnRgaCropHostFallbacks); + CHECK(restored.rknnCpuCropResizeFallbackCalls == original.rknnCpuCropResizeFallbackCalls); + CHECK(restored.rknnCpuCropResizeFallbackMs == original.rknnCpuCropResizeFallbackMs); CHECK(restored.rknnYolov8DirectCandidateCalls == original.rknnYolov8DirectCandidateCalls); CHECK(restored.rknnYolov8DirectCandidateFailures == original.rknnYolov8DirectCandidateFailures); CHECK(restored.rknnYolov8DirectPointsScanned == original.rknnYolov8DirectPointsScanned); diff --git a/test/test_message_system_handler.cc b/test/test_message_system_handler.cc index 1e70803ba..2099bb1ae 100644 --- a/test/test_message_system_handler.cc +++ b/test/test_message_system_handler.cc @@ -77,11 +77,19 @@ TEST_CASE("SystemHandler: QueryHardwareResource exposes accelerator preview tele CHECK(ret.resData.accelerator.mppEncodedFrames == preview.mpp_encoded_frames); CHECK(ret.resData.accelerator.mppDecodedFrames == preview.mpp_decoded_frames); CHECK(ret.resData.accelerator.mppCopyOutFrames == preview.mpp_copy_out_frames); + CHECK(ret.resData.accelerator.mppRgaCopyOutFrames == preview.mpp_rga_copy_out_frames); + CHECK(ret.resData.accelerator.mppRgaCopyInFrames == preview.mpp_rga_copy_in_frames); CHECK(ret.resData.accelerator.mppEarlyDroppedFrames == preview.mpp_early_dropped_frames); CHECK(ret.resData.accelerator.colorConvertFrames == inference.color_convert_frames); CHECK(ret.resData.accelerator.rknnForwards == inference.rknn_forwards); CHECK(ret.resData.accelerator.rknnDetectorForwards == inference.rknn_detector_forwards); CHECK(ret.resData.accelerator.rknnPreprocessFastHits == inference.rknn_preprocess_fast_hits); + CHECK(ret.resData.accelerator.rknnRgaCropResizeCalls == inference.rknn_rga_crop_resize_calls); + CHECK(ret.resData.accelerator.rknnRgaCropResizeFailures == inference.rknn_rga_crop_resize_failures); + CHECK(ret.resData.accelerator.rknnRgaCropDmaBufFrames == inference.rknn_rga_crop_dmabuf_frames); + CHECK(ret.resData.accelerator.rknnRgaCropHostFallbacks == inference.rknn_rga_crop_host_fallbacks); + CHECK(ret.resData.accelerator.rknnCpuCropResizeFallbackCalls == + inference.rknn_cpu_crop_resize_fallback_calls); CHECK(ret.resData.accelerator.rknnOutputsReleaseCalls == inference.rknn_outputs_release_calls); CHECK(ret.resData.accelerator.rknnNativeInt8Outputs == inference.rknn_native_int8_outputs); CHECK(ret.resData.accelerator.rknnBoundInputBindAttempts == inference.rknn_bound_input_bind_attempts); diff --git a/test/test_package_profile.py b/test/test_package_profile.py index 3260dc32f..e07b65099 100644 --- a/test/test_package_profile.py +++ b/test/test_package_profile.py @@ -9,6 +9,7 @@ import json import os import pathlib +import re import shutil import subprocess import tarfile @@ -396,6 +397,9 @@ def test_rk3576_release_builder_requires_pinned_rkllm(self) -> None: ) build = (REPOSITORY / "scripts/build_rknn.sh").read_text(encoding="utf-8") cmake = (REPOSITORY / "cmake/rkllm.cmake").read_text(encoding="utf-8") + media_cmake = (REPOSITORY / "cmake/rockchip_media.cmake").read_text( + encoding="utf-8" + ) self.assertIn( "image: ghcr.io/cosmo-wander-ai/cosmo_edge-build-env_rk3576@sha256:" @@ -409,9 +413,84 @@ def test_rk3576_release_builder_requires_pinned_rkllm(self) -> None: self.assertIn("878f9361fd3afa7e167b7079918918f78d2c1c2a", dockerfile) self.assertIn("install_rkllm_sdk.py", dockerfile) self.assertIn('lib/librkllmrt.so LICENSE', build) - self.assertIn("-DCOSMO_TARGET_CHIP=rk3576", build) + self.assertIn('-DCOSMO_TARGET_CHIP="${TARGET_CHIP}"', build) + self.assertIn('[-c rk3576|rv1126b]', build) self.assertIn('-DCOSMO_RKLLM_REQUIRED="${RKLLM_REQUIRED}"', build) self.assertIn('set(RKLLM_RUNTIME_LICENSE "${COSMO_RKLLM_ROOT}/LICENSE")', cmake) + self.assertIn("media_sysroot_lock.py", build) + self.assertIn("rockchip-media-manifest.json", media_cmake) + + def test_rknn_platform_profiles_share_backend_and_separate_artifacts(self) -> None: + rk3576 = json.loads( + (REPOSITORY / "config/rknn/platforms/rk3576.json").read_text( + encoding="utf-8" + ) + ) + rv1126b = json.loads( + (REPOSITORY / "config/rknn/platforms/rv1126b.json").read_text( + encoding="utf-8" + ) + ) + toolchain_lock = json.loads( + (REPOSITORY / "config/rknn/toolchain-lock.json").read_text( + encoding="utf-8" + ) + ) + for profile, chip in ((rk3576, "rk3576"), (rv1126b, "rv1126b")): + self.assertEqual(profile["backend"], "rknn") + self.assertEqual(profile["chip"], chip) + self.assertEqual(profile["conversion"]["target_platform"], chip) + self.assertTrue(profile["media"]["cpu_fallback"]) + self.assertEqual(profile["media"]["default_backend"], "rockchip") + self.assertEqual( + profile["media"]["runtime_lock"], + "../../rockchip-media/runtime-lock.json", + ) + self.assertTrue(profile["qualification"]["requires_target_bound_evidence"]) + self.assertEqual( + profile["qualification"]["status"], + toolchain_lock["qualification"][chip]["status"], + ) + self.assertNotEqual( + rk3576["media"]["runtime_profile"], + rv1126b["media"]["runtime_profile"], + ) + self.assertNotEqual( + rk3576["packaging"]["legacy_models_directory"], + rv1126b["packaging"]["legacy_models_directory"], + ) + + def test_shared_rknn_and_rockchip_sources_do_not_fork_by_chip(self) -> None: + source_roots = ( + REPOSITORY / "src/nn/device/rknn", + REPOSITORY / "src/media", + ) + chip_pattern = re.compile(r"\b(?:rk3576|rv1126b)\b", re.IGNORECASE) + source_files = sorted( + path + for root in source_roots + for path in root.rglob("*") + if path.suffix in {".cc", ".cpp", ".h", ".hpp"} + ) + + self.assertTrue(source_files) + for path in source_files: + relative = path.relative_to(REPOSITORY) + self.assertIsNone( + chip_pattern.search(path.name), + f"chip-specific backend source file is not allowed: {relative}", + ) + source = path.read_text(encoding="utf-8") + code_without_comments = re.sub( + r"//.*?$|/\*.*?\*/", + "", + source, + flags=re.MULTILINE | re.DOTALL, + ) + self.assertIsNone( + chip_pattern.search(code_without_comments), + f"chip-specific backend branching must move to platform data: {relative}", + ) if __name__ == "__main__": diff --git a/test/test_preview_pipeline_metrics.cc b/test/test_preview_pipeline_metrics.cc index dc4d8cafe..ea6c2babf 100644 --- a/test/test_preview_pipeline_metrics.cc +++ b/test/test_preview_pipeline_metrics.cc @@ -18,6 +18,12 @@ TEST_CASE("Preview pipeline metrics expose lifecycle and stage timings", "[media metrics.RecordMppDecodeFallback(); metrics.RecordMppCopyOut(true, 11'000'000); metrics.RecordMppCopyOut(false, 12'000'000); + metrics.RecordMppRgaCopyOut(true); + metrics.RecordMppRgaCopyOut(false); + metrics.RecordMppCpuCopyOutFallback(); + metrics.RecordMppRgaCopyIn(true); + metrics.RecordMppRgaCopyIn(false); + metrics.RecordMppCpuCopyInFallback(); metrics.RecordMppEarlyDrop(); metrics.PreviewFailed(); @@ -48,6 +54,12 @@ TEST_CASE("Preview pipeline metrics expose lifecycle and stage timings", "[media CHECK(during.mpp_copy_out_frames == 1); CHECK(during.mpp_copy_out_nanoseconds == 11'000'000); CHECK(during.mpp_copy_out_failures == 1); + CHECK(during.mpp_rga_copy_out_frames == 1); + CHECK(during.mpp_rga_copy_out_failures == 1); + CHECK(during.mpp_cpu_copy_out_fallbacks == 1); + CHECK(during.mpp_rga_copy_in_frames == 1); + CHECK(during.mpp_rga_copy_in_failures == 1); + CHECK(during.mpp_cpu_copy_in_fallbacks == 1); CHECK(during.mpp_early_dropped_frames == 1); metrics.PreviewStopped(false); diff --git a/test/test_rknn_fast_preprocess.cc b/test/test_rknn_fast_preprocess.cc index 1ac4d22a6..8cc9cc73c 100644 --- a/test/test_rknn_fast_preprocess.cc +++ b/test/test_rknn_fast_preprocess.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -64,7 +65,7 @@ TEST_CASE("RKNN detector fast preprocessing contracts are exact", "[nn][rknn][fa CHECK_FALSE(IsRknnDetectorResizeContract(224, 224, 1, {114, 114, 114})); CHECK(IsRknnNativeNormalizeContract({0.0f, 0.0f, 0.0f}, {}, 0.00392157f, {1, 640, 640, 3})); CHECK_FALSE(IsRknnNativeNormalizeContract({1.0f, 0.0f, 0.0f}, {}, 0.00392157f, {1, 640, 640, 3})); - CHECK_FALSE(IsRknnNativeNormalizeContract({0.0f, 0.0f, 0.0f}, {}, 0.00392157f, {1, 224, 224, 3})); + CHECK(IsRknnNativeNormalizeContract({0.0f, 0.0f, 0.0f}, {}, 0.00392157f, {1, 224, 224, 3})); const std::array rgb{0, 127, 255, 255, 1, 128}; std::array native{}; @@ -161,20 +162,9 @@ TEST_CASE("RKNN RGA bound input validates the native tensor and DMA-BUF target s std::string reason; CHECK(IsRknnRgaBoundInputCompatible(attr, 2, 2, &reason)); - rknn_tensor_attr uint8_attr{}; - REQUIRE(ConfigureRknnRgaUint8InputAttr(attr, 2, 2, uint8_attr, &reason)); - CHECK(uint8_attr.type == RKNN_TENSOR_UINT8); - CHECK(uint8_attr.fmt == RKNN_TENSOR_NHWC); - CHECK(uint8_attr.pass_through == 0); - CHECK(uint8_attr.qnt_type == attr.qnt_type); - CHECK(uint8_attr.zp == attr.zp); - CHECK(uint8_attr.scale == attr.scale); - CHECK(uint8_attr.size_with_stride == attr.size_with_stride); - CHECK(reason.empty()); - auto incompatible = attr; incompatible.zp = 0; - CHECK_FALSE(ConfigureRknnRgaUint8InputAttr(incompatible, 2, 2, uint8_attr, &reason)); + CHECK_FALSE(IsRknnRgaBoundInputCompatible(incompatible, 2, 2, &reason)); CHECK(reason.find("quantization") != std::string::npos); StubBoundInputProvider provider; @@ -211,6 +201,16 @@ TEST_CASE("RKNN RGA bound input requantizes UINT8 pixels in place without touchi CHECK(std::all_of(pixels.begin() + 18, pixels.end(), [](uint8_t value) { return value == 99; })); CHECK(reason.empty()); + std::array vector_sized{}; + std::iota(vector_sized.begin(), vector_sized.end(), uint8_t{0}); + auto expected = vector_sized; + std::transform(expected.begin(), expected.end(), expected.begin(), + [](uint8_t value) { return static_cast(value ^ 0x80); }); + REQUIRE(RequantizeRknnPackedUint8ToInt8InPlace(vector_sized.data(), vector_sized.size(), 1, 128, 1, 128, + &reason)); + CHECK(vector_sized == expected); + CHECK(std::string(RknnRgaBoundRequantizeImplementation()).find("xor-sign-bit") != std::string::npos); + CHECK_FALSE(RequantizeRknnPackedUint8ToInt8InPlace(pixels.data(), 8, 2, 2, 3, 2, &reason)); CHECK(reason.find("smaller") != std::string::npos); } @@ -277,6 +277,11 @@ TEST_CASE("RKNN core scheduling maps explicit and split modes deterministically" CHECK(ResolveRknnCoreMask(RknnCoreMode::Split, 0) == RKNN_NPU_CORE_0); CHECK(ResolveRknnCoreMask(RknnCoreMode::Split, 1) == RKNN_NPU_CORE_1); CHECK(ResolveRknnCoreMask(RknnCoreMode::Split, 2) == RKNN_NPU_CORE_0); + CHECK_FALSE(ShouldConfigureRknnCoreMask(RknnCoreMode::Auto)); + CHECK(ShouldConfigureRknnCoreMask(RknnCoreMode::Core0)); + CHECK(ShouldConfigureRknnCoreMask(RknnCoreMode::Core1)); + CHECK(ShouldConfigureRknnCoreMask(RknnCoreMode::Core01)); + CHECK(ShouldConfigureRknnCoreMask(RknnCoreMode::Split)); CHECK(std::string(RknnCoreModeName(RknnCoreMode::Core01)) == "core0_1"); } @@ -415,7 +420,7 @@ TEST_CASE("RKNN native output switch defaults on and supports explicit rollback" } } -TEST_CASE("RKNN classifier-sized normalization keeps the legacy float layout", +TEST_CASE("RKNN classifier-sized normalization uses the shared native INT8 layout", "[nn][rknn][fast-preprocess]") { using namespace cosmo::nn; Normalize normalize; @@ -427,16 +432,17 @@ TEST_CASE("RKNN classifier-sized normalization keeps the legacy float layout", node.SetSharedResource(&resource); node.LoadParam(&normalize); REQUIRE(bool(node.InferTopShapesWithBottoms({{1, 224, 224, 3}}, {DATA_TYPE_UINT8}))); - CHECK(node.GetTopBlobDataTypes().front() == DATA_TYPE_FLOAT); - CHECK((node.GetTopBlobShapes().front() == DimsVector{1, 3, 224, 224})); + CHECK(node.GetTopBlobDataTypes().front() == DATA_TYPE_INT8); + CHECK((node.GetTopBlobShapes().front() == DimsVector{1, 224, 224, 3})); + CHECK(resource.rknn_bound_input_preprocess_compatible); - auto bottom_desc = PackedImageDesc(224, 224, IMAGE_BGR); - bottom_desc.data_format = DATA_FORMAT_NCHW; // Legacy crop/resize nodes leave this metadata unset. - auto bottom = std::make_shared(bottom_desc, true); + auto bottom_desc = PackedImageDesc(224, 224, IMAGE_BGR); + auto bottom = std::make_shared(bottom_desc, true); BlobDesc top_desc; top_desc.device_type = DEVICE_NAIVE; - top_desc.data_type = DATA_TYPE_FLOAT; - top_desc.dims = {1, 3, 224, 224}; + top_desc.data_type = DATA_TYPE_INT8; + top_desc.data_format = DATA_FORMAT_NHWC; + top_desc.dims = {1, 224, 224, 3}; auto top = std::make_shared(top_desc, true); auto* input = static_cast(bottom->GetHandle().base); for (size_t pixel = 0; pixel < static_cast(224) * 224; ++pixel) { @@ -447,10 +453,10 @@ TEST_CASE("RKNN classifier-sized normalization keeps the legacy float layout", std::vector> bottoms{bottom}; std::vector> tops{top}; REQUIRE(bool(node.Forward(bottoms, tops))); - const auto* output = static_cast(top->GetHandle().base); - CHECK(output[0] == Catch::Approx(30.0f * 0.00392157f)); - CHECK(output[224 * 224] == Catch::Approx(20.0f * 0.00392157f)); - CHECK(output[2 * 224 * 224] == Catch::Approx(10.0f * 0.00392157f)); + const auto* output = static_cast(top->GetHandle().base); + CHECK(output[0] == -98); + CHECK(output[1] == -108); + CHECK(output[2] == -118); } TEST_CASE("RKNN normalize bypasses host mapping for an RGA-bound frame", "[nn][rknn][bound-input][rga]") { @@ -551,6 +557,119 @@ TEST_CASE("RKNN RGA preprocessing performs centered RGB letterbox on host buffer CHECK(output[center + 2] == 10); } +TEST_CASE("RKNN classifier crop-resize stages extreme scaling in RGA and emits packed RGB", + "[nn][rknn][rga][crop-resize]") { + using namespace cosmo::nn; + ScopedEnvironment no_force_fail("COSMO_RKNN_RGA_FORCE_FAIL", "0"); + CropResize crop; + crop.h_top_crop = {0.0f}; + crop.h_bottom_crop = {0.0f}; + crop.w_left_crop = {0.0f}; + crop.w_right_crop = {0.0f}; + crop.dsize = {224, 224}; + crop.gravity = 0; + crop.color = {114, 114, 114}; + + SharedResource resource; + RknnCropResizeNode node; + node.SetSharedResource(&resource); + node.LoadParam(&crop); + REQUIRE(bool(node.InferTopShapes())); + + auto image = std::make_shared(PackedImageDesc(8, 8, IMAGE_BGR), true); + auto* pixels = static_cast(image->GetHandle().base); + for (size_t pixel = 0; pixel < 64; ++pixel) { + pixels[pixel * 3] = 10; + pixels[pixel * 3 + 1] = 20; + pixels[pixel * 3 + 2] = 30; + } + BlobDesc rect_desc; + rect_desc.device_type = DEVICE_NAIVE; + rect_desc.data_type = DATA_TYPE_INT32; + rect_desc.dims = {1, 4}; + auto rect = std::make_shared(rect_desc, true); + auto* rect_data = static_cast(rect->GetHandle().base); + const std::array crop_rect{2, 2, 4, 4}; + std::copy(crop_rect.begin(), crop_rect.end(), rect_data); + + BlobDesc top_desc; + top_desc.device_type = DEVICE_NAIVE; + top_desc.data_type = DATA_TYPE_UINT8; + top_desc.data_format = DATA_FORMAT_NHWC; + top_desc.dims = node.GetTopBlobShapes().front(); + auto top = std::make_shared(top_desc, true); + const auto before = GetInferencePipelineMetrics().Snapshot(); + std::vector> images{image}; + std::vector> rects{rect}; + std::vector> tops{top}; + REQUIRE(bool(node.Forward(images, rects, tops))); + const auto after = GetInferencePipelineMetrics().Snapshot(); + CHECK(after.rknn_rga_crop_resize_calls == before.rknn_rga_crop_resize_calls + 2); + CHECK(after.rknn_rga_crop_resize_failures == before.rknn_rga_crop_resize_failures); + CHECK(after.rknn_rga_crop_host_fallbacks == before.rknn_rga_crop_host_fallbacks + 1); + CHECK(after.rknn_cpu_crop_resize_fallback_calls == before.rknn_cpu_crop_resize_fallback_calls); + CHECK(top->GetBlobDesc().image_format == IMAGE_RGB); + const auto* output = static_cast(top->GetHandle().base); + CHECK(output[0] == 30); + CHECK(output[1] == 20); + CHECK(output[2] == 10); +} + +TEST_CASE("RKNN classifier crop-resize keeps an exact CPU fallback", "[nn][rknn][rga][crop-resize]") { + using namespace cosmo::nn; + ScopedEnvironment force_fail("COSMO_RKNN_RGA_FORCE_FAIL", "1"); + CropResize crop; + crop.h_top_crop = {0.0f}; + crop.h_bottom_crop = {0.0f}; + crop.w_left_crop = {0.0f}; + crop.w_right_crop = {0.0f}; + crop.dsize = {4, 4}; + crop.gravity = 0; + + SharedResource resource; + RknnCropResizeNode node; + node.SetSharedResource(&resource); + node.LoadParam(&crop); + REQUIRE(bool(node.InferTopShapes())); + auto image = std::make_shared(PackedImageDesc(4, 4, IMAGE_BGR), true); + auto* pixels = static_cast(image->GetHandle().base); + for (size_t pixel = 0; pixel < 16; ++pixel) { + pixels[pixel * 3] = 10; + pixels[pixel * 3 + 1] = 20; + pixels[pixel * 3 + 2] = 30; + } + BlobDesc rect_desc; + rect_desc.device_type = DEVICE_NAIVE; + rect_desc.data_type = DATA_TYPE_INT32; + rect_desc.dims = {1, 4}; + auto rect = std::make_shared(rect_desc, true); + auto* rect_data = static_cast(rect->GetHandle().base); + rect_data[0] = 0; + rect_data[1] = 0; + rect_data[2] = 4; + rect_data[3] = 4; + BlobDesc top_desc; + top_desc.device_type = DEVICE_NAIVE; + top_desc.data_type = DATA_TYPE_UINT8; + top_desc.dims = node.GetTopBlobShapes().front(); + auto top = std::make_shared(top_desc, true); + + const auto before = GetInferencePipelineMetrics().Snapshot(); + std::vector> images{image}; + std::vector> rects{rect}; + std::vector> tops{top}; + REQUIRE(bool(node.Forward(images, rects, tops))); + const auto after = GetInferencePipelineMetrics().Snapshot(); + CHECK(after.rknn_rga_crop_resize_failures == before.rknn_rga_crop_resize_failures + 1); + CHECK(after.rknn_cpu_crop_resize_fallback_calls == before.rknn_cpu_crop_resize_fallback_calls + 1); + CHECK(top->GetBlobDesc().data_format == DATA_FORMAT_NHWC); + CHECK(top->GetBlobDesc().image_format == IMAGE_BGR); + const auto* output = static_cast(top->GetHandle().base); + CHECK(output[0] == 10); + CHECK(output[1] == 20); + CHECK(output[2] == 30); +} + TEST_CASE("RKNN RGA failure falls back once to CPU while preserving native input mapping", "[nn][rknn][rga][fast-preprocess]") { using namespace cosmo::nn; diff --git a/test/test_video_decoder_capability.cc b/test/test_video_decoder_capability.cc index ce8cc686e..b9d97f843 100644 --- a/test/test_video_decoder_capability.cc +++ b/test/test_video_decoder_capability.cc @@ -15,9 +15,9 @@ TEST_CASE("Video decoder capability uses a deterministic backend", "[media][deco CHECK_FALSE(h265.detail.empty()); #ifdef COSMO_MEDIA_USE_ROCKCHIP_BACKEND - CHECK(h264.backend == "rockchip-copy-out"); + CHECK(h264.backend == "rockchip-mpp-rga"); CHECK(h264.implementation == "rockchip-mpp-vpu"); - CHECK(h265.backend == "rockchip-copy-out"); + CHECK(h265.backend == "rockchip-mpp-rga"); CHECK(h265.implementation == "rockchip-mpp-vpu"); #elif defined(COSMO_MEDIA_USE_SOPHON_BACKEND) CHECK(h264.backend == "sophon-vpu"); diff --git a/test/test_video_encoder_capability.cc b/test/test_video_encoder_capability.cc index cadff3fa1..37c52615c 100644 --- a/test/test_video_encoder_capability.cc +++ b/test/test_video_encoder_capability.cc @@ -22,7 +22,7 @@ TEST_CASE("Video encoder capability uses a deterministic backend", "[media][enco "h264_nvenc")); #elif defined(COSMO_MEDIA_USE_ROCKCHIP_BACKEND) CHECK(capability.available); - CHECK(capability.backend == "rockchip-copy-first"); + CHECK(capability.backend == "rockchip-mpp-rga"); CHECK(capability.implementation == "rockchip-mpp"); const auto h265 = cosmo::media::VideoEncoder::Probe(cosmo::media::VideoCodecType::kH265); CHECK(h265.implementation != "rockchip-mpp"); diff --git a/tools/agent_workflow.py b/tools/agent_workflow.py index 704c18b1b..459b6236e 100755 --- a/tools/agent_workflow.py +++ b/tools/agent_workflow.py @@ -57,6 +57,7 @@ "device-deployment", } TPU_MLIR_OFFICIAL_REFERENCE = "https://github.com/sophgo/tpu-mlir#-installation" +RKNN_TOOLKIT2_OFFICIAL_REFERENCE = "https://github.com/airockchip/rknn-toolkit2" TOOLCHAIN_PROBE_SCRIPT = r""" import hashlib import importlib @@ -69,8 +70,10 @@ package = sys.argv[1] tool_paths = json.loads(sys.argv[2]) if len(sys.argv) > 2 else {} +family = sys.argv[3] if len(sys.argv) > 3 else "sophon" +module_name = sys.argv[4] if len(sys.argv) > 4 else package.replace("-", "_") distribution = importlib.metadata.distribution(package) -module = importlib.import_module(package.replace("-", "_")) +module = importlib.import_module(module_name) bin_dir = pathlib.Path(sys.executable).absolute().parent def tool(key, names, required=True): @@ -132,11 +135,15 @@ def record(path, resolution): if target.is_file(): item["targetSha256"] = hashlib.sha256(target.read_bytes()).hexdigest() runtime_links[relative] = item -required_runtime_links = { - "lib/libcmodel.so", - "lib/libbmlib.so", - "lib/libbmlib.so.0", -} +required_runtime_links = ( + { + "lib/libcmodel.so", + "lib/libbmlib.so", + "lib/libbmlib.so.0", + } + if family == "sophon" + else set() +) broken_required = [ relative + " -> " + target for relative, target in broken_links.items() @@ -147,7 +154,27 @@ def record(path, resolution): "installed distribution has broken required links: " + "; ".join(broken_required) ) +if family == "rknn": + rknn_class = getattr(module, "RKNN", None) + if rknn_class is None: + raise RuntimeError("rknn.api does not expose RKNN") + missing_methods = [ + name + for name in ("config", "load_onnx", "build", "export_rknn", "release") + if not callable(getattr(rknn_class, name, None)) + ] + if missing_methods: + raise RuntimeError("RKNN API is missing methods: " + ", ".join(missing_methods)) + tools = {} +else: + tools = { + "modelTransform": tool("modelTransform", ["model_transform.py", "model_transform"]), + "modelDeploy": tool("modelDeploy", ["model_deploy.py", "model_deploy"]), + "modelTool": tool("modelTool", ["model_tool"], required=False), + } + print(json.dumps({ + "family": family, "pythonExecutable": str(pathlib.Path(sys.executable).absolute()), "pythonVersion": sys.version.split()[0], "sysPrefix": str(pathlib.Path(sys.prefix).absolute()), @@ -158,11 +185,8 @@ def record(path, resolution): "version": distribution.version, "recordSha256": hashlib.sha256(record.read_bytes()).hexdigest(), }, - "tools": { - "modelTransform": tool("modelTransform", ["model_transform.py", "model_transform"]), - "modelDeploy": tool("modelDeploy", ["model_deploy.py", "model_deploy"]), - "modelTool": tool("modelTool", ["model_tool"], required=False), - }, + "module": {"name": module_name, "path": str(pathlib.Path(module.__file__).resolve())}, + "tools": tools, "runtimeLinks": runtime_links, "brokenOptionalLinks": broken_links, }, sort_keys=True)) @@ -483,10 +507,24 @@ def _objective_target_chip(contract: dict[str, Any]) -> str | None: if isinstance(mapped, str) and mapped.strip(): return mapped.strip().lower() objective = str(contract.get("userObjective", "")) - match = re.search(r"(?i)\b((?:bm|cv)\d+[a-z0-9]*)\b", objective) + match = re.search(r"(?i)\b((?:bm|cv|rk|rv)\d+[a-z0-9]*)\b", objective) return match.group(1).lower() if match else None +def _conversion_toolchain_family(target_chip: str | None) -> str | None: + """Map a target chip to its compiler family without implying product support.""" + normalized = str(target_chip or "").strip().lower() + if normalized.startswith(("rk", "rv")): + return "rknn" + if normalized.startswith(("bm", "cv")): + return "sophon" + return None + + +def _conversion_toolchain_label(family: str | None) -> str: + return "RKNN Toolkit2" if family == "rknn" else "TPU-MLIR" + + def _material_observations(contract: dict[str, Any], run_dir: Path) -> list[dict[str, Any]]: parameters = contract.get("parameters", {}) declared = parameters.get("sourceModel") @@ -592,12 +630,15 @@ def assess_task_report( _needs_input( "onnx-material", "当前物料不是 ONNX。请提供可用的 ONNX，或授权智能体把“从原训练工程导出 ONNX”作为单独交付阶段评估。", - "本版本只执行 ONNX 到 Sophon 产物的转换，不能把未实现的导出步骤当作已经支持。", + "本版本只执行 ONNX 到目标加速器产物的转换，不能把未实现的导出步骤当作已经支持。", required_before="route", ) ) target_chip = _objective_target_chip(contract) + toolchain_family = _conversion_toolchain_family(target_chip) or ( + "sophon" if not target_chip else None + ) if not target_chip: needs_input.append( _needs_input( @@ -606,6 +647,15 @@ def assess_task_report( "目标设备会改变产物，智能体不能仅凭示例替用户决定芯片映射。", ) ) + elif not toolchain_family: + needs_input.append( + _needs_input( + "target-toolchain", + "当前目标芯片尚未映射到仓库支持的模型编译工具链，请提供官方工具链依据或调整目标。", + "未知芯片不能默认套用 Sophon 或 Rockchip 的转换路径。", + required_before="route", + ) + ) host_os = str(inventory["host"]["os"]) host_arch = str(inventory["host"]["architecture"]).lower() @@ -616,32 +666,44 @@ def assess_task_report( str(environment.get("os", "")).lower() == "linux" and str(environment.get("architecture", "x86_64")).lower() in {"x86_64", "amd64"} ) - route_candidates.append( - { - "id": "local-linux-tpu-mlir", - "title": "在隔离的 Linux x86_64 开发环境中使用 TPU-MLIR", - "eligibility": "ELIGIBLE" if linux_local else "NEEDS_ENVIRONMENT", - "officialReference": TPU_MLIR_OFFICIAL_REFERENCE, - "detail": ( - "当前宿主满足官方工具链的操作系统与架构方向；后续仍需 doctor 核验实际能力。" - if linux_local - else "当前宿主不是 Linux x86_64；这不等于 CosmoEdge 不支持 Windows，而是该 Sophon 工具链路径需要 Linux。" - ), - } - ) - if not linux_local: + if toolchain_family: + route_suffix = "rknn-toolkit2" if toolchain_family == "rknn" else "tpu-mlir" + toolchain_label = _conversion_toolchain_label(toolchain_family) + official_reference = ( + RKNN_TOOLKIT2_OFFICIAL_REFERENCE + if toolchain_family == "rknn" + else TPU_MLIR_OFFICIAL_REFERENCE + ) route_candidates.append( { - "id": "remote-linux-tpu-mlir", - "title": "从当前机器编排隔离的 Linux x86_64 开发环境", - "eligibility": "ELIGIBLE" if remote_linux else "NEEDS_ENVIRONMENT", - "officialReference": TPU_MLIR_OFFICIAL_REFERENCE, - "detail": "当前机器可保留为材料整理和任务编排入口，转换在 Linux 开发环境执行。", + "id": f"local-linux-{route_suffix}", + "title": f"在隔离的 Linux x86_64 开发环境中使用 {toolchain_label}", + "eligibility": "ELIGIBLE" if linux_local else "NEEDS_ENVIRONMENT", + "officialReference": official_reference, + "detail": ( + "当前宿主满足已选工具链的操作系统与架构方向；后续仍需 doctor 核验实际能力。" + if linux_local + else ( + "当前宿主不是 Linux x86_64；这不等于 CosmoEdge 不支持 Windows，而是该 Sophon 工具链路径需要 Linux。" + if toolchain_family == "sophon" + else f"当前宿主不是 Linux x86_64；这不等于 CosmoEdge 不支持当前宿主，而是本次 {toolchain_label} 路径在隔离 Linux 环境执行。" + ) + ), } ) - recommended_route = ( - "local-linux-tpu-mlir" if linux_local else "remote-linux-tpu-mlir" - ) + if not linux_local: + route_candidates.append( + { + "id": f"remote-linux-{route_suffix}", + "title": f"从当前机器编排隔离的 Linux x86_64 {toolchain_label} 环境", + "eligibility": "ELIGIBLE" if remote_linux else "NEEDS_ENVIRONMENT", + "officialReference": official_reference, + "detail": "当前机器保留为材料整理和任务编排入口，转换在隔离 Linux 开发环境执行。", + } + ) + recommended_route = ( + f"local-linux-{route_suffix}" if linux_local else f"remote-linux-{route_suffix}" + ) required_grants: set[str] = set() if not linux_local and remote_linux: @@ -663,11 +725,12 @@ def assess_task_report( if not linux_local and not remote_linux: verdict = "NEEDS_ENVIRONMENT" + toolchain_label = _conversion_toolchain_label(toolchain_family) needs_input.append( _needs_input( "linux-development-environment", "请提供一台隔离的 Linux x86_64 开发环境，或允许智能体先给出可复用的 Docker/远程 Linux 准备方案；不要在生产设备上补环境。", - "Sophon TPU-MLIR 的官方开发路径以 Linux 环境为基础，当前宿主只适合作为编排入口。", + f"本次 {toolchain_label} 路径以隔离 Linux 环境为执行面，当前宿主只作为编排入口。", category="environment", required_before="doctor", ) @@ -802,6 +865,12 @@ def _resolve_executable(raw_value: str) -> str | None: def _toolchain_spec(parameters: dict[str, Any]) -> tuple[dict[str, Any] | None, str]: + family = _conversion_toolchain_family(str(parameters.get("targetChip", ""))) or "sophon" + default_package = "rknn-toolkit2" if family == "rknn" else "tpu_mlir" + default_module = "rknn.api" if family == "rknn" else "tpu_mlir" + official_reference = ( + RKNN_TOOLKIT2_OFFICIAL_REFERENCE if family == "rknn" else TPU_MLIR_OFFICIAL_REFERENCE + ) raw = parameters.get("toolchain") if not isinstance(raw, dict): if "toolchain" in parameters and raw is not None: @@ -810,23 +879,28 @@ def _toolchain_spec(parameters: dict[str, Any]) -> tuple[dict[str, Any] | None, if legacy: return None, ( "parameters.toolchainImage identifies only an execution image, not the " - "TPU-MLIR compiler package." + "complete compiler package." ) raw = {"kind": "auto"} kind = raw.get("kind", "auto") if kind not in TOOLCHAIN_KINDS: return None, "parameters.toolchain.kind must be auto, python-package, or container-image." - package = str(raw.get("package", "tpu_mlir")).strip() + package = str(raw.get("package", default_package)).strip() if not TOOLCHAIN_PACKAGE_PATTERN.fullmatch(package): return None, "parameters.toolchain.package contains unsupported characters." + module = str(raw.get("module", default_module)).strip() + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*", module): + return None, "parameters.toolchain.module must be a valid Python module name." version = raw.get("version") if version is not None and (not isinstance(version, str) or not version.strip()): return None, "parameters.toolchain.version must be a non-empty version requirement." normalized: dict[str, Any] = { + "family": family, "kind": kind, "package": package, + "module": module, "version": version.strip() if isinstance(version, str) else None, - "officialReference": TPU_MLIR_OFFICIAL_REFERENCE, + "officialReference": official_reference, } tool_paths = raw.get("toolPaths", {}) if not isinstance(tool_paths, dict) or any( @@ -866,13 +940,13 @@ def _parse_toolchain_probe( image: dict[str, Any] | None = None, ) -> tuple[dict[str, Any] | None, str]: if process.returncode != 0: - return None, _last_line(process.stderr) or "TPU-MLIR package inspection failed" + return None, _last_line(process.stderr) or "compiler package inspection failed" try: payload = json.loads(process.stdout) except json.JSONDecodeError: - return None, "TPU-MLIR package inspection returned unreadable output" + return None, "compiler package inspection returned unreadable output" if not isinstance(payload, dict): - return None, "TPU-MLIR package inspection returned an invalid identity" + return None, "compiler package inspection returned an invalid identity" identity: dict[str, Any] = {"kind": kind, **payload} if image is not None: identity["image"] = image @@ -884,6 +958,8 @@ def inspect_toolchain(specification: dict[str, Any]) -> tuple[dict[str, Any] | N package = str(specification["package"]) expected_version = specification.get("version") tool_paths = json.dumps(specification.get("toolPaths", {}), sort_keys=True) + family = str(specification.get("family", "sophon")) + module = str(specification.get("module", package.replace("-", "_"))) if specification["kind"] in {"auto", "python-package"}: declared = specification.get("pythonExecutable") candidates = [declared] if declared else [sys.executable, "python3"] @@ -901,7 +977,7 @@ def inspect_toolchain(specification: dict[str, Any]) -> tuple[dict[str, Any] | N continue seen_executables.add(executable_key) process = _run( - [executable, "-c", TOOLCHAIN_PROBE_SCRIPT, package, tool_paths], + [executable, "-c", TOOLCHAIN_PROBE_SCRIPT, package, tool_paths, family, module], timeout=90, ) identity, error = _parse_toolchain_probe(process, kind="python-package") @@ -933,6 +1009,8 @@ def inspect_toolchain(specification: dict[str, Any]) -> tuple[dict[str, Any] | N TOOLCHAIN_PROBE_SCRIPT, package, tool_paths, + family, + module, ], timeout=120, ) @@ -944,7 +1022,8 @@ def inspect_toolchain(specification: dict[str, Any]) -> tuple[dict[str, Any] | N if not identity: return None, error identity["officialReference"] = specification.get( - "officialReference", TPU_MLIR_OFFICIAL_REFERENCE + "officialReference", + RKNN_TOOLKIT2_OFFICIAL_REFERENCE if family == "rknn" else TPU_MLIR_OFFICIAL_REFERENCE, ) actual_version = str(identity.get("package", {}).get("version", "")) if not _version_satisfies(actual_version, expected_version): @@ -983,6 +1062,11 @@ def toolchain_environment(identity: dict[str, Any]) -> dict[str, str] | None: def _toolchain_tools_respond(identity: dict[str, Any]) -> tuple[bool, str]: + if identity.get("family") == "rknn": + module = identity.get("module", {}) + if isinstance(module, dict) and module.get("name") == "rknn.api": + return True, "" + return False, "admitted RKNN toolchain does not expose the rknn.api module" failures = [] for key in ("modelTransform", "modelDeploy"): tool = identity.get("tools", {}).get(key, {}) @@ -1081,7 +1165,9 @@ def _repository_compatibility_facts(project_root: Path) -> dict[str, Any]: header_path = project_root / "src" / "util" / "NnBackendConstants.h" guide_path = project_root / "docs" / "tutorials" / "05-model-porting" / "model-porting.md" resource_root = project_root / "data" / "resource" + rknn_profiles_root = project_root / "config" / "rknn" / "platforms" supported_chips: set[str] = set() + platform_profile_chips: set[str] = set() resource_chips: set[str] = set() guide_text = "" missing_sources: list[str] = [] @@ -1102,6 +1188,23 @@ def _repository_compatibility_facts(project_root: Path) -> dict[str, Any]: else: missing_sources.append("src/util/NnBackendConstants.h") + if rknn_profiles_root.is_dir(): + for profile_path in rknn_profiles_root.glob("*.json"): + try: + profile = load_json(profile_path) + except WorkflowError: + continue + if ( + isinstance(profile, dict) + and profile.get("backend") == "rknn" + and isinstance(profile.get("chip"), str) + ): + chip = profile["chip"].strip().upper() + supported_chips.add(chip) + platform_profile_chips.add(chip) + else: + missing_sources.append("config/rknn/platforms") + if resource_root.is_dir(): for config_path in resource_root.rglob("*.json"): try: @@ -1119,6 +1222,7 @@ def _repository_compatibility_facts(project_root: Path) -> dict[str, Any]: missing_sources.append("model-porting guide") return { "supportedChips": supported_chips, + "platformProfileChips": platform_profile_chips, "resourceChips": resource_chips, "guideText": guide_text, "missingSources": missing_sources, @@ -1189,16 +1293,24 @@ def compatibility_matrix_check( unknown = list(facts["missingSources"]) if target not in facts["supportedChips"]: unknown.append(f"code support for targetChip={target}") - if target not in facts["resourceChips"]: + if _conversion_toolchain_family(target) == "rknn": + if target not in facts["platformProfileChips"]: + unknown.append(f"an RKNN platform profile for targetChip={target}") + elif target not in facts["resourceChips"]: unknown.append(f"a repository resource example for targetChip={target}") guide_text = facts["guideText"] - if target not in guide_text or not all(token in guide_text for token in (".ONNX", ".BMODEL", "MODEL.NN")): + required_guide_tokens = ( + (".ONNX", ".RKNN") + if _conversion_toolchain_family(target) == "rknn" + else (".ONNX", ".BMODEL", "MODEL.NN") + ) + if target not in guide_text or not all(token in guide_text for token in required_guide_tokens): unknown.append(f"model-porting documentation for targetChip={target} and artifact types") if suffix == ".bmodel" and not artifact_chip: unknown.append("inspected chip metadata for the .bmodel artifact") elif suffix == ".nn" and not artifact_chip: unknown.append(artifact_source or "chip_type for the model.nn artifact") - elif suffix not in {".onnx", ".bmodel", ".nn"}: + elif suffix not in {".onnx", ".bmodel", ".nn", ".rknn"}: unknown.append(f"repository mapping for artifact type {artifact_type}") if unknown: return _check( @@ -1225,7 +1337,7 @@ def compatibility_matrix_check( "PASS", ( f"targetChip={target} matches the {artifact_detail}, toolchainChip={toolchain_chip}, " - "and repository code, resource, and model-porting facts." + "and repository code, platform/resource mapping, and model-porting facts." ), ) @@ -1247,6 +1359,12 @@ def task_environment_report( inventory = host_inventory(project_root) params = contract.get("parameters", {}) checks: list[dict[str, Any]] = [] + toolchain_family = ( + _conversion_toolchain_family(str(params.get("targetChip", ""))) + if task == "model-conversion" + else None + ) + toolchain_label = _conversion_toolchain_label(toolchain_family) architecture = inventory["host"]["architecture"].lower() host_os = str(inventory["host"]["os"]) @@ -1271,7 +1389,7 @@ def task_environment_report( _check( "C0", "FAIL", - f"The current host is {host_os}; the admitted Sophon conversion path executes on Linux x86_64.", + f"The current host is {host_os}; the admitted {toolchain_label} conversion path executes on Linux x86_64.", remediation=( "Keep this machine as the orchestration client and run assessment/doctor in an " "isolated Linux x86_64 development environment. Windows support elsewhere in " @@ -1407,7 +1525,7 @@ def task_environment_report( _check( "C5", "SKIP", - "TPU-MLIR capability admission is deferred to the selected Linux execution environment.", + f"{toolchain_label} capability admission is deferred to the selected Linux execution environment.", ) ) elif not toolchain_spec: @@ -1443,7 +1561,7 @@ def task_environment_report( "C5", "PASS", ( - f"Complete TPU-MLIR toolchain is frozen as {inspected['id']} " + f"Complete {toolchain_label} toolchain is frozen as {inspected['id']} " f"({inspected['package']['name']} " f"{inspected['package']['version']})." ), @@ -1454,7 +1572,7 @@ def task_environment_report( _check( "C5", "FAIL", - f"TPU-MLIR package exists but its compiler commands are unusable: {tools_error}.", + f"{toolchain_label} package exists but its conversion interface is unusable: {tools_error}.", remediation=( "Repair or replace the isolated compiler environment; do not treat " "package files alone as READY." @@ -1465,6 +1583,8 @@ def task_environment_report( ) else: official_base_only = ( + toolchain_family == "sophon" + and toolchain_spec["kind"] == "container-image" and str(toolchain_spec.get("image", "")).startswith("sophgo/tpuc_dev:") ) @@ -1476,7 +1596,7 @@ def task_environment_report( "The declared image is a base development environment, not a complete " f"TPU-MLIR compiler: {toolchain_error}." if official_base_only - else f"The declared TPU-MLIR toolchain is unavailable: {toolchain_error}." + else f"The declared {toolchain_label} toolchain is unavailable: {toolchain_error}." ), remediation=( "Add and freeze the TPU-MLIR package in that environment or select an " @@ -1752,8 +1872,14 @@ def create_task_run( material_paths.append(destination.relative_to(run_dir).as_posix()) parameters: dict[str, Any] = {} - if task == "model-conversion" and len(material_paths) == 1: - parameters["sourceModel"] = material_paths[0] + if task == "model-conversion": + model_material_paths = [ + path + for path in material_paths + if Path(path).suffix.lower() in {".onnx", ".pt", ".pth", ".mlir", ".bmodel", ".nn"} + ] + if len(model_material_paths) == 1: + parameters["sourceModel"] = model_material_paths[0] if target_chip: parameters["targetChip"] = target_chip.strip().lower() if remote_linux: diff --git a/tools/check_onnx_model.py b/tools/check_onnx_model.py index 67c8d29a5..203752de4 100755 --- a/tools/check_onnx_model.py +++ b/tools/check_onnx_model.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Validate an ONNX model and run a zero-input ONNX Runtime smoke inference.""" +"""Validate an ONNX model, optionally with an ONNX Runtime smoke inference.""" from __future__ import annotations @@ -73,23 +73,86 @@ def _safe_shape(value: Any) -> list[Any]: return list(value) if isinstance(value, (list, tuple)) else [] -def inspect_model(model_path: Path, overrides: dict[str, list[int]]) -> dict[str, Any]: +def _declared_value_info(onnx: Any, value: Any) -> dict[str, Any]: + tensor_type = value.type.tensor_type + dimensions: list[Any] = [] + for dimension in tensor_type.shape.dim: + if dimension.HasField("dim_value"): + dimensions.append(int(dimension.dim_value)) + elif dimension.dim_param: + dimensions.append(dimension.dim_param) + else: + dimensions.append(None) + try: + element_type = onnx.TensorProto.DataType.Name(tensor_type.elem_type).lower() + except ValueError: + element_type = str(tensor_type.elem_type) + return { + "name": value.name, + "declaredShape": dimensions, + "type": f"tensor({element_type})", + } + + +def inspect_model( + model_path: Path, + overrides: dict[str, list[int]], + *, + run_runtime: bool = True, +) -> dict[str, Any]: try: - import numpy as np import onnx - import onnxruntime as ort except ImportError as error: raise OnnxCheckError( - "missing dependency; use an approved environment containing numpy, onnx, and onnxruntime" + "missing dependency; use an approved environment containing onnx" ) from error try: model = onnx.load(str(model_path)) onnx.checker.check_model(model) - session = ort.InferenceSession(str(model_path), providers=["CPUExecutionProvider"]) except Exception as error: raise OnnxCheckError(f"ONNX load/check failed: {error}") from error + model_identity = { + "path": model_path.name, + "sha256": sha256_file(model_path), + "sizeBytes": model_path.stat().st_size, + } + graph = { + "irVersion": int(model.ir_version), + "opsets": [ + {"domain": item.domain or "ai.onnx", "version": int(item.version)} + for item in model.opset_import + ], + } + if not run_runtime: + if overrides: + raise OnnxCheckError("shape overrides require ONNX Runtime smoke validation") + return { + "schemaVersion": "1.0", + "status": "PASS", + "validationMode": "checker-only", + "model": model_identity, + "graph": graph, + "dependencies": {"onnx": onnx.__version__}, + "inputs": [_declared_value_info(onnx, value) for value in model.graph.input], + "outputs": [_declared_value_info(onnx, value) for value in model.graph.output], + "note": "ONNX checker validates source structure; runtime execution is checked after target conversion-input normalization.", + } + + try: + import numpy as np + import onnxruntime as ort + except ImportError as error: + raise OnnxCheckError( + "missing runtime dependency; use an approved environment containing numpy and onnxruntime" + ) from error + + try: + session = ort.InferenceSession(str(model_path), providers=["CPUExecutionProvider"]) + except Exception as error: + raise OnnxCheckError(f"ONNX Runtime load failed: {error}") from error + inputs = [] feed = {} total_elements = 0 @@ -139,11 +202,9 @@ def inspect_model(model_path: Path, overrides: dict[str, list[int]]) -> dict[str return { "schemaVersion": "1.0", "status": "PASS", - "model": { - "path": model_path.name, - "sha256": sha256_file(model_path), - "sizeBytes": model_path.stat().st_size, - }, + "validationMode": "runtime-smoke", + "model": model_identity, + "graph": graph, "providers": session.get_providers(), "dependencies": { "numpy": np.__version__, @@ -167,7 +228,10 @@ def write_result(result: dict[str, Any], destination: str | None) -> None: "outputs:", [(item["name"], item["declaredShape"], item["type"]) for item in result["outputs"]], ) - print("runtime output shapes:", [item["runtimeShape"] for item in result["outputs"]]) + if result.get("validationMode") == "runtime-smoke": + print("runtime output shapes:", [item["runtimeShape"] for item in result["outputs"]]) + else: + print("validation mode:", result.get("validationMode", "checker-only")) return payload = json.dumps(result, ensure_ascii=False, indent=2) + "\n" if destination == "-": @@ -199,13 +263,20 @@ def main(arguments: list[str] | None = None) -> int: metavar="PATH", help="write JSON to PATH, or stdout when PATH is omitted", ) + parser.add_argument( + "--checker-only", + action="store_true", + help="run onnx.checker without importing or executing ONNX Runtime", + ) options = parser.parse_args(arguments) model_path = Path(options.model).expanduser().resolve() if not model_path.is_file(): parser.error(f"model does not exist: {model_path}") overrides = dict(options.shape) + if options.checker_only and overrides: + parser.error("--shape cannot be combined with --checker-only") try: - result = inspect_model(model_path, overrides) + result = inspect_model(model_path, overrides, run_runtime=not options.checker_only) except OnnxCheckError as error: failure = { "schemaVersion": "1.0", diff --git a/tools/conversion_workflow_dispatch.py b/tools/conversion_workflow_dispatch.py new file mode 100644 index 000000000..c6ed94d12 --- /dev/null +++ b/tools/conversion_workflow_dispatch.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Dispatch the admitted model-conversion workflow by backend family.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import agent_workflow as core +import model_conversion_workflow as sophon_conversion + + +def _contract_argument(arguments: list[str]) -> str: + for index, argument in enumerate(arguments): + if argument == "--contract" and index + 1 < len(arguments): + return arguments[index + 1] + if argument.startswith("--contract="): + return argument.split("=", 1)[1] + raise core.WorkflowError("--contract is required") + + +def workflow_family(arguments: list[str]) -> str: + contract_path, _, contract = core.resolve_contract_context( + Path(_contract_argument(arguments)) + ) + del contract_path + target_chip = str(contract.get("parameters", {}).get("targetChip", "")) + family = core._conversion_toolchain_family(target_chip) + if family not in {"sophon", "rknn"}: + raise core.WorkflowError( + f"no conversion workflow is registered for targetChip={target_chip or 'unspecified'}" + ) + return family + + +def main(arguments: list[str] | None = None) -> int: + args = list(sys.argv[1:] if arguments is None else arguments) + if not args or args[0] not in {"convert", "verify"}: + print("usage: conversion_workflow_dispatch.py convert|verify --contract ...", file=sys.stderr) + return 2 + try: + family = workflow_family(args[1:]) + if family == "rknn": + from rknn import agent_conversion_workflow as rknn_conversion + + return rknn_conversion.main(args) + return sophon_conversion.main(args) + except core.WorkflowError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/rknn/agent_conversion_workflow.py b/tools/rknn/agent_conversion_workflow.py new file mode 100644 index 000000000..856f97457 --- /dev/null +++ b/tools/rknn/agent_conversion_workflow.py @@ -0,0 +1,962 @@ +#!/usr/bin/env python3 +"""Measured ONNX-to-RKNN conversion and evidence workflow.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +import time +from pathlib import Path +from typing import Any + +import agent_workflow as core +import model_conversion_workflow as common + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +VIDEO_SUFFIXES = {".mp4", ".mov", ".mkv", ".avi"} + + +def _repository_file(raw_value: Any, default: str, allowed_root: Path, field: str) -> Path: + value = str(raw_value or default).strip() + candidate = (PROJECT_ROOT / value).resolve() + try: + candidate.relative_to(allowed_root.resolve()) + except ValueError as error: + raise core.WorkflowError(f"{field} must stay under {allowed_root.relative_to(PROJECT_ROOT)}") from error + if not candidate.is_file(): + raise core.WorkflowError(f"{field} does not exist: {value}") + return candidate + + +def _calibration_source(parameters: dict[str, Any], run_dir: Path) -> Path | None: + if parameters["quantization"] != "INT8": + return None + raw = parameters["raw"].get("calibrationSource") + if raw: + return core.resolve_run_input(run_dir, str(raw)) + candidates = sorted( + path for path in (run_dir / "inputs").iterdir() if path.suffix.lower() in VIDEO_SUFFIXES + ) + if len(candidates) != 1: + raise core.WorkflowError( + "INT8 RKNN conversion requires parameters.calibrationSource or exactly one input video" + ) + return candidates[0].resolve() + + +def conversion_parameters(contract: dict[str, Any], run_dir: Path) -> dict[str, Any]: + parameters = common.conversion_parameters(contract, run_dir) + if core._conversion_toolchain_family(parameters["targetChip"]) != "rknn": + raise core.WorkflowError("RKNN conversion requires an RK/RV target chip") + if str(parameters["raw"].get("outputKind", "")).lower() != "rknn": + raise core.WorkflowError("RKNN conversion requires parameters.outputKind=rknn") + if parameters["quantization"] not in {"INT8", "F16", "FP16"}: + raise core.WorkflowError("RKNN conversion supports INT8 or FP16 in this workflow") + + model_name = parameters["modelName"] + target_chip = parameters["targetChip"] + spec_path = _repository_file( + parameters["raw"].get("modelSpec"), + f"config/rknn/models/{model_name}.json", + PROJECT_ROOT / "config" / "rknn" / "models", + "parameters.modelSpec", + ) + profile_path = _repository_file( + parameters["raw"].get("platformProfile"), + f"config/rknn/platforms/{target_chip}.json", + PROJECT_ROOT / "config" / "rknn" / "platforms", + "parameters.platformProfile", + ) + spec = core.load_json(spec_path) + profile = core.load_json(profile_path) + if not isinstance(spec, dict) or spec.get("name") != model_name: + raise core.WorkflowError("RKNN model spec name does not match parameters.modelName") + if not isinstance(profile, dict) or profile.get("backend") != "rknn": + raise core.WorkflowError("RKNN platform profile must declare backend=rknn") + if str(profile.get("chip", "")).lower() != target_chip: + raise core.WorkflowError("RKNN platform profile chip does not match parameters.targetChip") + if str(profile.get("conversion", {}).get("target_platform", "")).lower() != target_chip: + raise core.WorkflowError("RKNN platform profile conversion target does not match the task") + toolchain_lock_raw = profile.get("toolchain_lock") + if not isinstance(toolchain_lock_raw, str) or not toolchain_lock_raw.strip(): + raise core.WorkflowError("RKNN platform profile must reference a toolchain lock") + toolchain_lock_path = (profile_path.parent / toolchain_lock_raw).resolve() + toolchain_lock_root = (PROJECT_ROOT / "config" / "rknn").resolve() + try: + toolchain_lock_path.relative_to(toolchain_lock_root) + except ValueError as error: + raise core.WorkflowError("RKNN toolchain lock must stay under config/rknn") from error + if not toolchain_lock_path.is_file(): + raise core.WorkflowError("RKNN platform profile toolchain lock does not exist") + toolchain_lock = core.load_json(toolchain_lock_path) + if not isinstance(toolchain_lock, dict) or toolchain_lock.get("family") != "rknn-toolkit2": + raise core.WorkflowError("RKNN platform profile references an invalid toolchain lock") + requested_version = parameters["toolchainSpec"].get("version") + if requested_version and not core._version_satisfies( + str(toolchain_lock.get("version", "")), requested_version + ): + raise core.WorkflowError("RKNN toolchain lock conflicts with the task contract") + + person_detector = None + person_detector_spec_path = None + person_detector_spec = None + if spec.get("model_type") == "classify": + detector_name = spec.get("calibration", {}).get("person_detector_model") + if not isinstance(detector_name, str) or not detector_name.strip(): + raise core.WorkflowError( + "classifier model spec must declare calibration.person_detector_model" + ) + person_detector_spec_path = _repository_file( + parameters["raw"].get("personDetectorSpec"), + f"config/rknn/models/{detector_name}.json", + PROJECT_ROOT / "config" / "rknn" / "models", + "parameters.personDetectorSpec", + ) + person_detector_spec = core.load_json(person_detector_spec_path) + if ( + not isinstance(person_detector_spec, dict) + or person_detector_spec.get("name") != detector_name + or person_detector_spec.get("model_type") != "yolov8_det" + ): + raise core.WorkflowError( + "classifier calibration person-detector spec is invalid" + ) + detector_source = person_detector_spec.get("source_repository_path") + if not isinstance(detector_source, str) or not detector_source.strip(): + raise core.WorkflowError( + "classifier calibration person-detector spec has no source path" + ) + person_detector = _repository_file( + parameters["raw"].get("personDetector"), + detector_source, + PROJECT_ROOT / "data" / "resource", + "parameters.personDetector", + ) + parameters.update( + { + "specPath": spec_path, + "spec": spec, + "profilePath": profile_path, + "profile": profile, + "toolchainLockPath": toolchain_lock_path, + "toolchainLock": toolchain_lock, + "calibrationSource": _calibration_source(parameters, run_dir), + "personDetector": person_detector, + "personDetectorSpecPath": person_detector_spec_path, + "personDetectorSpec": person_detector_spec, + } + ) + return parameters + + +def _python_path(parameters: dict[str, Any]) -> str: + value = parameters["pythonExecutable"] + resolved = shutil.which(value) if not Path(value).is_absolute() else value + if not resolved or not Path(resolved).is_file(): + raise core.WorkflowError(f"Python executable is unavailable: {value}") + return str(resolved) + + +def _run_stage( + command: list[str], + *, + cwd: Path, + log_path: Path, + commands: list[str], + run_dir: Path, + environment: dict[str, str] | None, + detail: str, + timeout: int = 1800, +) -> None: + process = common._run_logged( + command, + cwd=cwd, + log_path=log_path, + commands=commands, + run_dir=run_dir, + timeout=timeout, + environment=environment, + ) + if process.returncode != 0: + raise common.ExecutionFailure(detail) + + +def _onnx_check_command( + python: str, + model_path: Path, + report_path: Path, + *, + checker_only: bool, +) -> list[str]: + command = [ + python, + str(PROJECT_ROOT / "tools" / "check_onnx_model.py"), + str(model_path), + ] + if checker_only: + command.append("--checker-only") + command.extend(["--json", str(report_path)]) + return command + + +def _prepare_conversion_input( + parameters: dict[str, Any], + *, + python: str, + attempt_dir: Path, + logs_dir: Path, + commands: list[str], + run_dir: Path, + environment: dict[str, str] | None, +) -> tuple[Path, list[dict[str, Any]]]: + source = parameters["sourceModel"] + spec = parameters["spec"] + source_hash = core.sha256_file(source) + expected_source_hash = spec.get("source_sha256") + if expected_source_hash and source_hash != expected_source_hash: + raise core.WorkflowError( + f"RKNN model spec source SHA-256 differs from {common._run_relative(source, run_dir)}" + ) + expected_input_hash = spec.get("conversion", {}).get("input_sha256", expected_source_hash) + if not expected_input_hash or expected_input_hash == source_hash: + return source, [] + + adapter = spec.get("conversion", {}).get("output_adapter") + if adapter != "yolo_dfl_6head_v1": + raise core.WorkflowError( + "model spec requires a transformed RKNN input but declares no supported transform" + ) + converted = attempt_dir / f"{parameters['modelName']}-opset.onnx" + converted_report = attempt_dir / "opset-conversion.json" + convert_command = [ + python, + str(PROJECT_ROOT / "tools" / "rknn" / "convert_onnx_opset.py"), + "--input", + str(source), + "--output", + str(converted), + "--opset", + str(spec["conversion"]["maximum_onnx_opset"]), + "--report", + str(converted_report), + ] + maximum_ir = spec["conversion"].get("maximum_onnx_ir_version") + if maximum_ir is not None: + convert_command.extend(["--ir-version", str(maximum_ir)]) + _run_stage( + convert_command, + cwd=attempt_dir, + log_path=logs_dir / "S2-opset-conversion.log", + commands=commands, + run_dir=run_dir, + environment=environment, + detail="ONNX opset conversion failed", + ) + + extracted = attempt_dir / f"{parameters['modelName']}-runtime.onnx" + extracted_report = attempt_dir / "output-extraction.json" + _run_stage( + [ + python, + str(PROJECT_ROOT / "tools" / "rknn" / "extract_yolov8_heads.py"), + "--input", + str(converted), + "--output", + str(extracted), + "--report", + str(extracted_report), + ], + cwd=attempt_dir, + log_path=logs_dir / "S2-output-extraction.log", + commands=commands, + run_dir=run_dir, + environment=environment, + detail="YOLOv8 output-head extraction failed", + ) + actual_hash = core.sha256_file(extracted) + if actual_hash != expected_input_hash: + raise common.ExecutionFailure( + f"transformed RKNN input SHA-256 mismatch: expected {expected_input_hash}, got {actual_hash}" + ) + return extracted, [ + common._artifact(converted_report, run_dir, "opset-provenance"), + common._artifact(extracted_report, run_dir, "output-adapter-provenance"), + ] + + +def _prepare_calibration_person_detector( + parameters: dict[str, Any], + *, + python: str, + attempt_dir: Path, + verification_dir: Path, + logs_dir: Path, + commands: list[str], + run_dir: Path, + environment: dict[str, str] | None, +) -> tuple[Path | None, dict[str, Any] | None]: + source = parameters["personDetector"] + if source is None: + return None, None + spec = parameters["personDetectorSpec"] + spec_path = parameters["personDetectorSpecPath"] + expected_source_hash = spec.get("source_sha256") + source_hash = core.sha256_file(source) + if expected_source_hash and source_hash != expected_source_hash: + raise core.WorkflowError( + "classifier calibration person-detector source SHA-256 differs from its model spec" + ) + conversion = spec.get("conversion", {}) + maximum_opset = conversion.get("maximum_onnx_opset") + maximum_ir = conversion.get("maximum_onnx_ir_version") + if not isinstance(maximum_opset, int) or not isinstance(maximum_ir, int): + raise core.WorkflowError( + "classifier calibration person-detector spec must bound ONNX opset and IR versions" + ) + + normalized = attempt_dir / "person-detector-runtime.onnx" + normalization_report = attempt_dir / "person-detector-normalization.json" + _run_stage( + [ + python, + str(PROJECT_ROOT / "tools" / "rknn" / "convert_onnx_opset.py"), + "--input", + str(source), + "--output", + str(normalized), + "--opset", + str(maximum_opset), + "--ir-version", + str(maximum_ir), + "--report", + str(normalization_report), + ], + cwd=attempt_dir, + log_path=logs_dir / "S3-person-detector-normalization.log", + commands=commands, + run_dir=run_dir, + environment=environment, + detail="classifier calibration person-detector normalization failed", + timeout=300, + ) + runtime_report = verification_dir / "person-detector-runtime-check.json" + _run_stage( + _onnx_check_command( + python, + normalized, + runtime_report, + checker_only=False, + ), + cwd=PROJECT_ROOT, + log_path=logs_dir / "S3-person-detector-runtime-check.log", + commands=commands, + run_dir=run_dir, + environment=environment, + detail="normalized classifier calibration person detector failed runtime validation", + timeout=300, + ) + return normalized, { + "source": { + "path": source.relative_to(PROJECT_ROOT).as_posix(), + "sha256": source_hash, + }, + "spec": { + "path": spec_path.relative_to(PROJECT_ROOT).as_posix(), + "sha256": core.sha256_file(spec_path), + }, + "normalizedInput": common._artifact( + normalized, run_dir, "calibration-person-detector" + ), + "normalizationReport": common._artifact( + normalization_report, run_dir, "calibration-person-detector-provenance" + ), + "runtimeReport": common._artifact( + runtime_report, run_dir, "calibration-person-detector-runtime-check" + ), + } + + +def execute_conversion( + contract_path: Path, run_dir: Path, contract: dict[str, Any] +) -> tuple[dict[str, Any], dict[str, Any]]: + parameters = conversion_parameters(contract, run_dir) + environment_report = common._read_environment_report(contract_path, run_dir, contract) + selection = common._read_asset_selection(run_dir) + admitted_toolchain = environment_report["toolchain"] + current_toolchain, error = core.inspect_toolchain(parameters["toolchainSpec"]) + if not current_toolchain: + raise core.WorkflowError(f"admitted RKNN toolchain is no longer available: {error}") + if current_toolchain.get("id") != admitted_toolchain.get("id"): + raise core.WorkflowError("RKNN toolchain identity changed after admission; rerun doctor") + if str(current_toolchain.get("package", {}).get("version", "")) != str( + parameters["toolchainLock"].get("version", "") + ): + raise core.WorkflowError("admitted RKNN Toolkit2 version differs from the platform lock") + + attempt, previous_attempts = common._archive_previous_attempt(run_dir) + work_dir = run_dir / "work" / f"attempt-{attempt}" + artifacts_dir = run_dir / "artifacts" / f"attempt-{attempt}" + logs_dir = run_dir / "logs" / f"attempt-{attempt}" + verification_dir = run_dir / "verification" / f"attempt-{attempt}" + for directory in (work_dir, artifacts_dir, logs_dir, verification_dir): + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + + manifest_path = run_dir / "execution-manifest.json" + commands: list[str] = [] + started = time.monotonic() + manifest: dict[str, Any] = { + "schemaVersion": "1.0", + "status": "RUNNING", + "attempt": attempt, + "previousAttempts": previous_attempts, + "runId": contract["runId"], + "task": contract["task"], + "commit": environment_report.get("commit"), + "tree": environment_report.get("tree"), + "repository": environment_report["repository"], + "contractSha256": core.sha256_file(contract_path), + "routeAssessment": "route-assessment.json", + "routeAssessmentSha256": environment_report["routeAssessmentSha256"], + "startedAt": common.utc_now(), + "source": common._artifact(parameters["sourceModel"], run_dir, "source-onnx"), + "target": { + "backend": parameters["targetBackend"], + "chip": parameters["targetChip"], + "toolchainChip": parameters["toolchainChip"], + "quantization": parameters["quantization"], + "inputLayout": parameters["inputLayout"], + "inputShapes": parameters["inputShapes"], + "expectedOutputShapes": parameters["expectedOutputShapes"], + }, + "modelSpec": { + "path": parameters["specPath"].relative_to(PROJECT_ROOT).as_posix(), + "sha256": core.sha256_file(parameters["specPath"]), + }, + "platformProfile": { + "path": parameters["profilePath"].relative_to(PROJECT_ROOT).as_posix(), + "sha256": core.sha256_file(parameters["profilePath"]), + }, + "toolchainLock": { + "path": parameters["toolchainLockPath"].relative_to(PROJECT_ROOT).as_posix(), + "sha256": core.sha256_file(parameters["toolchainLockPath"]), + }, + "toolchain": current_toolchain, + "dataFlow": common._data_flow_record(contract), + "selectedAssets": core.redact_data(selection["selectedAssets"]), + "assetDifferences": core.redact_data(selection["differences"]), + "commands": commands, + "stages": {}, + "artifacts": [], + } + core.atomic_write_json(manifest_path, manifest) + + environment = core.toolchain_environment(current_toolchain) + python = _python_path(parameters) + try: + preflight_path = verification_dir / "onnx-check.json" + _run_stage( + _onnx_check_command( + python, + parameters["sourceModel"], + preflight_path, + checker_only=True, + ), + cwd=PROJECT_ROOT, + log_path=logs_dir / "S1-onnx-preflight.log", + commands=commands, + run_dir=run_dir, + environment=environment, + detail="ONNX preflight failed", + timeout=300, + ) + manifest["stages"]["onnxPreflight"] = { + "status": "PASS", + "report": common._run_relative(preflight_path, run_dir), + "reportSha256": core.sha256_file(preflight_path), + } + core.atomic_write_json(manifest_path, manifest) + + conversion_input, transform_reports = _prepare_conversion_input( + parameters, + python=python, + attempt_dir=work_dir, + logs_dir=logs_dir, + commands=commands, + run_dir=run_dir, + environment=environment, + ) + runtime_check_path = verification_dir / "conversion-input-runtime-check.json" + _run_stage( + _onnx_check_command( + python, + conversion_input, + runtime_check_path, + checker_only=False, + ), + cwd=PROJECT_ROOT, + log_path=logs_dir / "S2-conversion-input-runtime-check.log", + commands=commands, + run_dir=run_dir, + environment=environment, + detail="normalized RKNN conversion input failed ONNX Runtime smoke validation", + timeout=300, + ) + manifest["stages"]["transform"] = { + "status": "PASS", + "input": common._artifact(conversion_input, run_dir, "rknn-conversion-input"), + "reports": transform_reports, + "runtimeReport": common._run_relative(runtime_check_path, run_dir), + "runtimeReportSha256": core.sha256_file(runtime_check_path), + } + core.atomic_write_json(manifest_path, manifest) + + dataset_path = None + if parameters["quantization"] == "INT8": + person_detector, person_detector_evidence = ( + _prepare_calibration_person_detector( + parameters, + python=python, + attempt_dir=work_dir, + verification_dir=verification_dir, + logs_dir=logs_dir, + commands=commands, + run_dir=run_dir, + environment=environment, + ) + ) + calibration_dir = work_dir / "calibration" + calibration_command = [ + python, + str(PROJECT_ROOT / "tools" / "rknn" / "prepare_validation_data.py"), + "--spec", + str(parameters["specPath"]), + "--video", + str(parameters["calibrationSource"]), + "--output-dir", + str(calibration_dir), + "--samples", + str(parameters["spec"]["calibration"]["minimum_samples"]), + ] + if person_detector is not None: + calibration_command.extend( + ["--person-detector", str(person_detector)] + ) + _run_stage( + calibration_command, + cwd=work_dir, + log_path=logs_dir / "S3-calibration.log", + commands=commands, + run_dir=run_dir, + environment=environment, + detail="representative calibration preparation failed", + ) + dataset_path = calibration_dir / "dataset.txt" + calibration_manifest = calibration_dir / "manifest.json" + manifest["stages"]["calibration"] = { + "status": "PASS", + "dataset": common._artifact(dataset_path, run_dir, "calibration-dataset"), + "manifest": common._artifact( + calibration_manifest, run_dir, "calibration-manifest" + ), + "personDetector": person_detector_evidence, + } + else: + manifest["stages"]["calibration"] = { + "status": "SKIP", + "detail": "FP16 RKNN conversion does not use an INT8 calibration dataset.", + } + core.atomic_write_json(manifest_path, manifest) + + precision = "int8" if parameters["quantization"] == "INT8" else "fp16" + artifact_path = artifacts_dir / f"{parameters['modelName']}_{parameters['targetChip']}_{precision}.rknn" + build_report_path = verification_dir / "rknn-build.json" + build_command = [ + python, + str(PROJECT_ROOT / "tools" / "rknn" / "convert_model.py"), + "--spec", + str(parameters["specPath"]), + "--platform-profile", + str(parameters["profilePath"]), + "--model", + str(conversion_input), + "--output", + str(artifact_path), + "--report", + str(build_report_path), + ] + if parameters["quantization"] == "INT8": + build_command.extend(["--quantize", "--dataset", str(dataset_path)]) + _run_stage( + build_command, + cwd=work_dir, + log_path=logs_dir / "S4-rknn-build.log", + commands=commands, + run_dir=run_dir, + environment=environment, + detail="RKNN Toolkit2 build failed", + timeout=3600, + ) + artifact = common._artifact( + artifact_path, run_dir, f"{parameters['targetChip']}-rknn" + ) + manifest["stages"]["deploy"] = {"status": "PASS", "artifact": artifact} + manifest["stages"]["modelInfo"] = { + "status": "PASS", + "contractMatches": True, + "report": common._run_relative(build_report_path, run_dir), + "reportSha256": core.sha256_file(build_report_path), + } + manifest["stages"]["tensorCompare"] = { + "status": "UNVERIFIED", + "detail": "Numerical parity is deferred to the target-bound RKNN runtime validation.", + } + manifest["artifacts"] = [artifact] + manifest["status"] = "COMPLETE" + manifest["completedAt"] = common.utc_now() + manifest["durationSeconds"] = round(time.monotonic() - started, 3) + core.atomic_write_json(manifest_path, manifest) + return manifest, parameters + except (core.WorkflowError, common.ExecutionFailure) as error: + common._failed_manifest(manifest_path, manifest, str(error)) + raise + + +def _manifest_file(run_dir: Path, entry: Any, role: str) -> Path: + if not isinstance(entry, dict) or not isinstance(entry.get("path"), str): + raise core.WorkflowError(f"execution manifest has no {role} path") + path = core.resolve_run_input(run_dir, entry["path"]) + if core.sha256_file(path) != entry.get("sha256"): + raise core.WorkflowError(f"execution manifest {role} SHA-256 changed") + return path + + +def verify_conversion( + contract_path: Path, run_dir: Path, contract: dict[str, Any] +) -> dict[str, Any]: + parameters = conversion_parameters(contract, run_dir) + environment = common._read_environment_report(contract_path, run_dir, contract) + selection = common._read_asset_selection(run_dir) + manifest_path = run_dir / "execution-manifest.json" + manifest = core.load_json(manifest_path) + if not isinstance(manifest, dict) or manifest.get("status") != "COMPLETE": + raise core.WorkflowError("execution-manifest.json is absent or not COMPLETE") + if manifest.get("runId") != contract["runId"]: + raise core.WorkflowError("execution manifest belongs to another run") + if manifest.get("contractSha256") != core.sha256_file(contract_path): + raise core.WorkflowError("execution manifest does not match the current contract") + + stages: list[dict[str, str]] = [] + try: + source = _manifest_file(run_dir, manifest.get("source"), "source") + preflight = manifest.get("stages", {}).get("onnxPreflight", {}) + preflight_report = _manifest_file( + run_dir, + {"path": preflight.get("report"), "sha256": preflight.get("reportSha256")}, + "ONNX preflight report", + ) + preflight_data = core.load_json(preflight_report) + source_ok = ( + isinstance(preflight_data, dict) + and preflight.get("status") == "PASS" + and preflight_data.get("status") == "PASS" + and preflight_data.get("validationMode") == "checker-only" + and preflight_data.get("model", {}).get("sha256") == core.sha256_file(source) + ) + except core.WorkflowError: + source_ok = False + stages.append( + { + "id": "S1", + "status": "PASS" if source_ok else "FAIL", + "detail": "ONNX source checker and source identity match." if source_ok else "ONNX source-check evidence is missing or changed.", + } + ) + + transform = manifest.get("stages", {}).get("transform", {}) + try: + conversion_input = _manifest_file(run_dir, transform.get("input"), "conversion input") + runtime_report = _manifest_file( + run_dir, + { + "path": transform.get("runtimeReport"), + "sha256": transform.get("runtimeReportSha256"), + }, + "conversion-input runtime report", + ) + runtime_data = core.load_json(runtime_report) + expected_hash = parameters["spec"].get("conversion", {}).get( + "input_sha256", parameters["spec"].get("source_sha256") + ) + input_hash = core.sha256_file(conversion_input) + transform_ok = ( + transform.get("status") == "PASS" + and (not expected_hash or input_hash == expected_hash) + and isinstance(runtime_data, dict) + and runtime_data.get("status") == "PASS" + and runtime_data.get("validationMode") == "runtime-smoke" + and runtime_data.get("model", {}).get("sha256") == input_hash + ) + except core.WorkflowError: + transform_ok = False + stages.append( + { + "id": "S2", + "status": "PASS" if transform_ok else "FAIL", + "detail": "The target-independent model contract resolves to a runtime-checked RKNN conversion input." if transform_ok else "RKNN conversion-input identity or runtime evidence is missing or changed.", + } + ) + + calibration = manifest.get("stages", {}).get("calibration", {}) + if parameters["quantization"] == "INT8": + try: + _manifest_file(run_dir, calibration.get("dataset"), "calibration dataset") + calibration_manifest_path = _manifest_file( + run_dir, calibration.get("manifest"), "calibration manifest" + ) + calibration_data = core.load_json(calibration_manifest_path) + if not isinstance(calibration_data, dict): + raise core.WorkflowError("calibration manifest must be an object") + person_detector_ok = calibration_data.get("person_detector") is None + if parameters["personDetector"] is not None: + detector = calibration.get("personDetector") + if not isinstance(detector, dict): + raise core.WorkflowError( + "calibration person-detector evidence is missing" + ) + normalized_detector = _manifest_file( + run_dir, + detector.get("normalizedInput"), + "calibration person-detector input", + ) + normalization_report_path = _manifest_file( + run_dir, + detector.get("normalizationReport"), + "calibration person-detector normalization report", + ) + detector_runtime_report_path = _manifest_file( + run_dir, + detector.get("runtimeReport"), + "calibration person-detector runtime report", + ) + normalization_data = core.load_json(normalization_report_path) + detector_runtime_data = core.load_json(detector_runtime_report_path) + if not isinstance(normalization_data, dict) or not isinstance( + detector_runtime_data, dict + ): + raise core.WorkflowError( + "calibration person-detector reports must be objects" + ) + normalized_hash = core.sha256_file(normalized_detector) + expected_output_shape = parameters["personDetectorSpec"]["outputs"][0][ + "shape" + ] + output_shapes = [ + item.get("runtimeShape") + for item in detector_runtime_data.get("outputs", []) + if isinstance(item, dict) + ] + source_record = detector.get("source", {}) + spec_record = detector.get("spec", {}) + person_detector_ok = ( + source_record.get("path") + == parameters["personDetector"].relative_to(PROJECT_ROOT).as_posix() + and source_record.get("sha256") + == core.sha256_file(parameters["personDetector"]) + and spec_record.get("path") + == parameters["personDetectorSpecPath"] + .relative_to(PROJECT_ROOT) + .as_posix() + and spec_record.get("sha256") + == core.sha256_file(parameters["personDetectorSpecPath"]) + and normalization_data.get("source", {}).get("sha256") + == source_record.get("sha256") + and normalization_data.get("converted", {}).get("sha256") + == normalized_hash + and detector_runtime_data.get("status") == "PASS" + and detector_runtime_data.get("validationMode") == "runtime-smoke" + and detector_runtime_data.get("model", {}).get("sha256") + == normalized_hash + and expected_output_shape in output_shapes + and calibration_data.get("person_detector", {}).get("sha256") + == normalized_hash + ) + calibration_ok = ( + calibration.get("status") == "PASS" + and person_detector_ok + and len(calibration_data.get("samples", [])) + >= int(parameters["spec"]["calibration"]["minimum_samples"]) + ) + except (core.WorkflowError, KeyError, TypeError): + calibration_ok = False + stages.append( + { + "id": "S3", + "status": "PASS" if calibration_ok else "FAIL", + "detail": "Representative INT8 calibration inputs and identities are recorded." if calibration_ok else "INT8 calibration evidence is incomplete or changed.", + } + ) + else: + stages.append( + {"id": "S3", "status": "SKIP", "detail": "FP16 conversion does not require calibration."} + ) + + artifact_entry = manifest.get("artifacts", [None])[0] + model_info = manifest.get("stages", {}).get("modelInfo", {}) + try: + artifact_path = _manifest_file(run_dir, artifact_entry, "RKNN artifact") + build_report_path = _manifest_file( + run_dir, + {"path": model_info.get("report"), "sha256": model_info.get("reportSha256")}, + "RKNN build report", + ) + build_report = core.load_json(build_report_path) + artifact_ok = ( + isinstance(build_report, dict) + and model_info.get("status") == "PASS" + and model_info.get("contractMatches") is True + and build_report.get("build", {}).get("target_platform") == parameters["targetChip"] + and build_report.get("artifact", {}).get("sha256") == core.sha256_file(artifact_path) + and build_report.get("spec", {}).get("sha256") == core.sha256_file(parameters["specPath"]) + and build_report.get("platform_profile", {}).get("sha256") + == core.sha256_file(parameters["profilePath"]) + and manifest.get("toolchainLock", {}).get("sha256") + == core.sha256_file(parameters["toolchainLockPath"]) + ) + except (core.WorkflowError, IndexError, TypeError): + artifact_path = None + artifact_ok = False + stages.append( + { + "id": "S4", + "status": "PASS" if artifact_ok else "FAIL", + "detail": "RKNN artifact, target profile, model contract, toolchain report, and hashes agree." if artifact_ok else "RKNN artifact contract evidence is incomplete or changed.", + } + ) + stages.append( + { + "id": "S5", + "status": "UNVERIFIED", + "detail": "Target runtime, numerical parity, video pipeline, and business acceptance remain separate device gates.", + } + ) + + required_ids = {"S1", "S2", "S4"} + if parameters["quantization"] == "INT8": + required_ids.add("S3") + required = [item for item in stages if item["id"] in required_ids] + development_verdict = "COMPLETE" if all(item["status"] == "PASS" for item in required) else "FAILED" + deliverables = [] + if artifact_path is not None: + deliverables.append( + { + "path": artifact_entry["path"], + "sha256": artifact_entry["sha256"], + "sizeBytes": artifact_path.stat().st_size, + "role": artifact_entry.get("role", "rknn"), + } + ) + evidence = { + "schemaVersion": "1.0", + "runId": contract["runId"], + "task": contract["task"], + "userObjective": core.redact_text(contract["userObjective"]), + "commit": environment.get("commit"), + "timestamp": common.utc_now(), + "environmentReport": "environment-report.json", + "routeAssessment": "route-assessment.json", + "routeAssessmentSha256": environment["routeAssessmentSha256"], + "contractSha256": core.sha256_file(contract_path), + "authorityGrants": sorted(core._authority_grants(contract)), + "selectedAssets": core.redact_data(selection["selectedAssets"]), + "assetDifferences": core.redact_data(selection["differences"]), + "attempts": [ + *manifest.get("previousAttempts", []), + common._attempt_summary(manifest), + ], + "dataFlow": core.redact_data(manifest.get("dataFlow", {})), + "stages": stages, + "deliverables": deliverables, + "developmentVerdict": development_verdict, + "promotionVerdict": "NOT_REQUESTED", + "deviceVerdict": "NOT_RUN", + "pendingOnDevice": [ + "RKNN runtime import and NPU execution", + "ONNX-to-RKNN numerical comparison", + "one-stream video, OSD, rule, and alarm loop", + "stability and resource measurements", + ], + "commands": [core.redact_text(str(item)) for item in manifest.get("commands", [])], + } + core.atomic_write_json(run_dir / "evidence.json", evidence) + lines = [ + "# RKNN model conversion evidence", + "", + f"- Development conversion: **{development_verdict}**", + "- Device result: **NOT_RUN**", + "", + "## Layered verification", + "", + *[f"- {item['id']} {item['status']}: {item['detail']}" for item in stages], + "", + "## Deliverables", + "", + *[ + f"- `{item['path']}` — SHA-256 `{item['sha256']}` ({item['sizeBytes']} bytes)" + for item in deliverables + ], + "", + "Conversion completion does not imply RV1126B device or production acceptance.", + "", + ] + common._write_private_text(run_dir / "evidence.md", "\n".join(lines)) + return evidence + + +def convert_main(arguments: list[str]) -> int: + parser = argparse.ArgumentParser(prog="convert_model.sh") + parser.add_argument("--contract", required=True) + options = parser.parse_args(arguments) + contract_path, run_dir, contract = core.resolve_contract_context(options.contract) + try: + manifest, _ = execute_conversion(contract_path, run_dir, contract) + print(f"Conversion artifact: {manifest['artifacts'][0]['path']}") + print("Execution manifest: execution-manifest.json") + return 0 + except common.ExecutionFailure as error: + print(f"conversion failed: {error}", file=sys.stderr) + return 1 + + +def verify_main(arguments: list[str]) -> int: + parser = argparse.ArgumentParser(prog="verify.sh") + parser.add_argument("--contract", required=True) + options = parser.parse_args(arguments) + contract_path, run_dir, contract = core.resolve_contract_context(options.contract) + evidence = verify_conversion(contract_path, run_dir, contract) + print(f"Development conversion: {evidence['developmentVerdict']}") + print("Device validation: NOT_RUN") + return 0 if evidence["developmentVerdict"] == "COMPLETE" else 1 + + +def main(arguments: list[str] | None = None) -> int: + args = list(sys.argv[1:] if arguments is None else arguments) + if not args: + print("usage: agent_conversion_workflow.py convert|verify ...", file=sys.stderr) + return 2 + command = args.pop(0) + try: + if command == "convert": + return convert_main(args) + if command == "verify": + return verify_main(args) + raise core.WorkflowError(f"unknown command: {command}") + except core.WorkflowError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/rknn/convert_model.py b/tools/rknn/convert_model.py index f121ae96c..873b60d14 100755 --- a/tools/rknn/convert_model.py +++ b/tools/rknn/convert_model.py @@ -30,6 +30,11 @@ def require_success(code: int, action: str) -> None: def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--spec", required=True, type=Path) + parser.add_argument( + "--platform-profile", + type=Path, + help="RKNN platform profile; supplies the chip-specific target_platform", + ) parser.add_argument("--model", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--dataset", type=Path, help="RKNN calibration dataset list") @@ -40,11 +45,14 @@ def main() -> int: args = parser.parse_args() spec_path = args.spec.resolve() + platform_profile_path = args.platform_profile.resolve() if args.platform_profile else None model_path = args.model.resolve() output_path = args.output.resolve() report_path = (args.report or output_path.with_suffix(output_path.suffix + ".build.json")).resolve() if not spec_path.is_file() or not model_path.is_file(): parser.error("--spec and --model must be existing files") + if platform_profile_path is not None and not platform_profile_path.is_file(): + parser.error("--platform-profile must be an existing file") if args.quantize and (args.dataset is None or not args.dataset.is_file()): parser.error("--quantize requires an existing --dataset list") if not args.quantize and args.dataset is not None: @@ -54,6 +62,25 @@ def main() -> int: parser.error(f"refusing to overwrite {candidate}; pass --force") spec = json.loads(spec_path.read_text(encoding="utf-8")) + conversion = spec["conversion"] + profile = None + if platform_profile_path is not None: + profile = json.loads(platform_profile_path.read_text(encoding="utf-8")) + if profile.get("backend") != "rknn": + raise RuntimeError("platform profile backend must be rknn") + target_platform = profile.get("conversion", {}).get("target_platform") + if not isinstance(target_platform, str) or not target_platform: + raise RuntimeError("platform profile must define conversion.target_platform") + legacy_target = conversion.get("target_platform") + if legacy_target and legacy_target != target_platform: + raise RuntimeError( + f"model spec target_platform={legacy_target} conflicts with platform profile " + f"target_platform={target_platform}" + ) + else: + target_platform = conversion.get("target_platform") + if not isinstance(target_platform, str) or not target_platform: + parser.error("--platform-profile is required when the model spec is target-independent") expected_hash = spec["conversion"].get("input_sha256", spec.get("source_sha256")) actual_hash = sha256(model_path) if expected_hash and actual_hash != expected_hash: @@ -83,7 +110,6 @@ def main() -> int: if spec["conversion"].get("preprocessing_owner") != "host": raise RuntimeError("P0-P4 contract requires preprocessing_owner=host") - conversion = spec["conversion"] output_path.parent.mkdir(parents=True, exist_ok=True) rknn = RKNN(verbose=args.verbose) try: @@ -91,7 +117,7 @@ def main() -> int: # NCHW float tensors and must remain the single preprocessing owner. require_success( rknn.config( - target_platform=conversion["target_platform"], + target_platform=target_platform, optimization_level=int(conversion["optimization_level"]), ), "rknn.config", @@ -111,6 +137,15 @@ def main() -> int: "created_at": datetime.now(timezone.utc).isoformat(), "rknn_toolkit2_version": importlib.metadata.version("rknn-toolkit2"), "spec": {"path": str(spec_path), "sha256": sha256(spec_path)}, + "platform_profile": ( + { + "path": str(platform_profile_path), + "sha256": sha256(platform_profile_path), + "chip": profile.get("chip"), + } + if platform_profile_path is not None and profile is not None + else None + ), "source": { "path": str(model_path), "sha256": actual_hash, @@ -119,7 +154,7 @@ def main() -> int: "outputs": [item.name for item in model.graph.output], }, "build": { - "target_platform": conversion["target_platform"], + "target_platform": target_platform, "optimization_level": conversion["optimization_level"], "quantized": args.quantize, "dataset": str(args.dataset.resolve()) if args.dataset else None, diff --git a/tools/rknn/cosmo_rknn_fastpath_qualify.cc b/tools/rknn/cosmo_rknn_fastpath_qualify.cc index ce58ba58d..d61156797 100644 --- a/tools/rknn/cosmo_rknn_fastpath_qualify.cc +++ b/tools/rknn/cosmo_rknn_fastpath_qualify.cc @@ -192,17 +192,18 @@ struct PathBuffers { class QualificationRunner { public: QualificationRunner(const std::vector& model, int source_height, int source_width, - bool qualify_rga_bound_input, bool qualify_rga_uint8_input, bool uint8_input_contract) + bool isolate_rga_input, bool qualify_rga_bound_native_input, + bool uint8_input_contract) : source_height_(source_height), source_width_(source_width), - qualify_rga_bound_input_(qualify_rga_bound_input) { + qualify_rga_bound_input_(isolate_rga_input) { ConfigurePreprocessing(); if (uint8_input_contract) network_.SetInputContract(cosmo::nn::kRknnRgbUint8InputContract); ConfigureNetwork(network_, model, nullptr, output_); - if (qualify_rga_bound_input_) { + if (isolate_rga_input) { bound_network_ = std::make_unique(); - if (qualify_rga_uint8_input) + if (qualify_rga_bound_native_input) bound_network_->SetInputContract(cosmo::nn::kRknnRgbUint8InputContract); ConfigureNetwork(*bound_network_, model, &fast_resource_, bound_output_); } @@ -433,7 +434,7 @@ int main(int argc, char** argv) { std::cerr << "Usage: " << argv[0] << " " "[--direct-output-parity|--direct-output-parity-uint8-contract|" - "--rga-bound-input-parity|--rga-bound-uint8-parity]\n"; + "--rga-host-preprocess-parity|--rga-bound-native-int8-parity]\n"; return 2; } @@ -448,9 +449,9 @@ int main(int argc, char** argv) { qualification_option == "--direct-output-parity-uint8-contract"; const bool qualify_direct_output_uint8 = qualification_option == "--direct-output-parity-uint8-contract"; - const bool qualify_rga_bound_input = argc == 7 && std::string(argv[6]) == "--rga-bound-input-parity"; - const bool qualify_rga_uint8_input = argc == 7 && std::string(argv[6]) == "--rga-bound-uint8-parity"; - if (argc == 7 && !qualify_direct_output && !qualify_rga_bound_input && !qualify_rga_uint8_input) + const bool qualify_rga_host_input = qualification_option == "--rga-host-preprocess-parity"; + const bool qualify_rga_bound_native_input = qualification_option == "--rga-bound-native-int8-parity"; + if (argc == 7 && !qualify_direct_output && !qualify_rga_host_input && !qualify_rga_bound_native_input) throw std::runtime_error("unknown qualification option"); if (height <= 0 || width <= 0) throw std::runtime_error("source dimensions must be positive"); @@ -463,8 +464,9 @@ int main(int argc, char** argv) { LogGuard log_guard; const auto metrics_before = cosmo::nn::GetInferencePipelineMetrics().Snapshot(); - QualificationRunner runner(model, height, width, qualify_rga_bound_input || qualify_rga_uint8_input, - qualify_rga_uint8_input, qualify_direct_output_uint8); + QualificationRunner runner(model, height, width, + qualify_rga_host_input || qualify_rga_bound_native_input, + qualify_rga_bound_native_input, qualify_direct_output_uint8); std::unique_ptr direct_output_runner; if (qualify_direct_output) direct_output_runner = @@ -533,7 +535,7 @@ int main(int argc, char** argv) { metrics_before.rknn_yolov8_score_sum_points_rejected << '\n'; } - if (qualify_direct_output || qualify_rga_bound_input || qualify_rga_uint8_input) { + if (qualify_direct_output || qualify_rga_host_input || qualify_rga_bound_native_input) { const auto metrics_after = cosmo::nn::GetInferencePipelineMetrics().Snapshot(); std::cout << "bound_input_bind_attempts=" << metrics_after.rknn_bound_input_bind_attempts - @@ -601,6 +603,8 @@ int main(int argc, char** argv) { << metrics_after.rknn_rga_bound_requantize_failures - metrics_before.rknn_rga_bound_requantize_failures << '\n'; + std::cout << "rga_bound_requantize_implementation=" + << cosmo::nn::RknnRgaBoundRequantizeImplementation() << '\n'; std::cout << "rga_bound_input_normalize_bypasses=" << metrics_after.rknn_rga_bound_input_normalize_bypasses - metrics_before.rknn_rga_bound_input_normalize_bypasses diff --git a/tools/rknn/media_sysroot_lock.py b/tools/rknn/media_sysroot_lock.py new file mode 100755 index 000000000..e746b8a14 --- /dev/null +++ b/tools/rknn/media_sysroot_lock.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Seal and verify candidate-bound Rockchip MPP/RGA sysroots.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any + + +MANIFEST_NAME = ".cosmo-rockchip-media.json" + + +class MediaSysrootError(RuntimeError): + """Raised when a Rockchip media sysroot violates its selected lock.""" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_json(path: Path, field: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise MediaSysrootError(f"cannot read {field}: {path}: {error}") from error + if not isinstance(value, dict): + raise MediaSysrootError(f"{field} must contain a JSON object: {path}") + return value + + +def safe_relative_path(raw: str, field: str) -> PurePosixPath: + path = PurePosixPath(raw) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise MediaSysrootError(f"{field} must be a safe relative path: {raw}") + return path + + +def load_runtime( + profile_path: Path, +) -> tuple[dict[str, Any], dict[str, Any], str, dict[str, Any]]: + profile_path = profile_path.expanduser().resolve() + profile = load_json(profile_path, "platform profile") + media = profile.get("media") + if not isinstance(media, dict): + raise MediaSysrootError("platform profile has no media object") + runtime_id = media.get("runtime_profile") + lock_value = media.get("runtime_lock") + if not isinstance(runtime_id, str) or not runtime_id: + raise MediaSysrootError("platform profile has no media.runtime_profile") + if not isinstance(lock_value, str) or not lock_value: + raise MediaSysrootError("platform profile has no media.runtime_lock") + + lock_relative = Path(lock_value) + if lock_relative.is_absolute(): + raise MediaSysrootError("media.runtime_lock must be relative to the profile") + lock_path = profile_path.parent.joinpath(lock_relative).resolve() + try: + lock_path.relative_to(profile_path.parents[2]) + except (IndexError, ValueError) as error: + raise MediaSysrootError("media.runtime_lock must stay under config/") from error + lock = load_json(lock_path, "Rockchip media runtime lock") + runtimes = lock.get("runtimes") + if not isinstance(runtimes, dict) or runtime_id not in runtimes: + raise MediaSysrootError( + f"runtime profile {runtime_id!r} is absent from {lock_path}" + ) + runtime = runtimes[runtime_id] + if not isinstance(runtime, dict): + raise MediaSysrootError(f"runtime profile {runtime_id!r} must be an object") + return profile, media, runtime_id, runtime + + +def inspect_elf(path: Path) -> dict[str, str]: + readelf = shutil.which("readelf") + if not readelf: + raise MediaSysrootError("readelf is required to verify Rockchip libraries") + header = subprocess.run( + [readelf, "-h", str(path)], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + dynamic = subprocess.run( + [readelf, "-d", str(path)], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + if header.returncode or dynamic.returncode: + detail = (header.stderr or dynamic.stderr).strip() + raise MediaSysrootError(f"readelf rejected {path}: {detail}") + machine_match = re.search(r"^\s*Machine:\s*(.+?)\s*$", header.stdout, re.M) + soname_match = re.search(r"\(SONAME\).*?\[(.+?)\]", dynamic.stdout) + if not machine_match or not soname_match: + raise MediaSysrootError(f"cannot determine ELF machine and SONAME: {path}") + return { + "machine": machine_match.group(1), + "soname": soname_match.group(1), + } + + +def _inside_root(root: Path, path: Path, field: str) -> None: + try: + path.resolve().relative_to(root) + except ValueError as error: + raise MediaSysrootError(f"{field} escapes the media sysroot: {path}") from error + + +def inspect_artifacts(root: Path, runtime: dict[str, Any]) -> dict[str, Any]: + artifacts = runtime.get("artifacts") + if not isinstance(artifacts, dict) or not artifacts: + raise MediaSysrootError("runtime profile has no artifacts") + records: dict[str, Any] = {} + for raw_path, expected_value in sorted(artifacts.items()): + if not isinstance(raw_path, str) or not isinstance(expected_value, dict): + raise MediaSysrootError("runtime artifacts must map paths to objects") + relative = safe_relative_path(raw_path, "runtime artifact") + path = root.joinpath(*relative.parts) + if not path.is_file(): + raise MediaSysrootError(f"Rockchip media artifact is missing: {path}") + _inside_root(root, path, "runtime artifact") + record: dict[str, Any] = { + "sha256": sha256(path), + "size_bytes": path.stat().st_size, + } + expected_sha = expected_value.get("sha256") + if expected_sha and record["sha256"] != expected_sha: + raise MediaSysrootError( + f"Rockchip media artifact hash mismatch: {raw_path}: " + f"{record['sha256']} != {expected_sha}" + ) + expected_elf = expected_value.get("elf") + if expected_elf is not None: + if not isinstance(expected_elf, dict): + raise MediaSysrootError(f"ELF lock must be an object: {raw_path}") + actual_elf = inspect_elf(path) + for field in ("machine", "soname"): + expected = expected_elf.get(field) + if expected and actual_elf[field] != expected: + raise MediaSysrootError( + f"Rockchip media ELF {field} mismatch: {raw_path}: " + f"{actual_elf[field]} != {expected}" + ) + record["elf"] = actual_elf + records[raw_path] = record + + expected_links = runtime.get("links", {}) + if not isinstance(expected_links, dict): + raise MediaSysrootError("runtime links must be an object") + links: dict[str, str] = {} + for raw_path, expected_target in sorted(expected_links.items()): + if not isinstance(raw_path, str) or not isinstance(expected_target, str): + raise MediaSysrootError("runtime links must map paths to targets") + relative = safe_relative_path(raw_path, "runtime link") + safe_relative_path(expected_target, "runtime link target") + path = root.joinpath(*relative.parts) + if not path.is_symlink(): + raise MediaSysrootError(f"Rockchip media link is missing: {path}") + actual_target = os.readlink(path) + if actual_target != expected_target: + raise MediaSysrootError( + f"Rockchip media link mismatch: {raw_path}: " + f"{actual_target} != {expected_target}" + ) + _inside_root(root, path, "runtime link") + links[raw_path] = actual_target + return {"artifacts": records, "links": links} + + +def expected_source_revisions(runtime: dict[str, Any]) -> dict[str, str]: + sources = runtime.get("sources") + if not isinstance(sources, dict) or not sources: + raise MediaSysrootError("runtime profile has no sources") + revisions: dict[str, str] = {} + for name, value in sorted(sources.items()): + if not isinstance(name, str) or not isinstance(value, dict): + raise MediaSysrootError("runtime sources must map names to objects") + revision = value.get("revision") + if not isinstance(revision, str) or not revision: + raise MediaSysrootError(f"runtime source has no revision: {name}") + revisions[name] = revision + return revisions + + +def seal_sysroot( + profile_path: Path, root: Path, supplied_sources: dict[str, str] +) -> Path: + _, media, runtime_id, runtime = load_runtime(profile_path) + if not media.get("require_sealed_sysroot"): + raise MediaSysrootError( + "selected platform runtime does not require a sealed sysroot" + ) + expected_sources = expected_source_revisions(runtime) + if supplied_sources != expected_sources: + raise MediaSysrootError( + f"source revisions do not match {runtime_id}: " + f"{supplied_sources} != {expected_sources}" + ) + root = root.expanduser().resolve() + if not root.is_dir(): + raise MediaSysrootError(f"media sysroot does not exist: {root}") + inspected = inspect_artifacts(root, runtime) + manifest = { + "schema_version": 1, + "runtime_profile": runtime_id, + "created_at": datetime.now(timezone.utc).isoformat(), + "sources": runtime["sources"], + **inspected, + } + manifest_path = root / MANIFEST_NAME + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return manifest_path + + +def verify_sysroot(profile_path: Path, root: Path) -> dict[str, Any]: + _, media, runtime_id, runtime = load_runtime(profile_path) + root = root.expanduser().resolve() + if not root.is_dir(): + raise MediaSysrootError(f"media sysroot does not exist: {root}") + inspected = inspect_artifacts(root, runtime) + manifest_path = root / MANIFEST_NAME + if not manifest_path.is_file(): + if media.get("require_sealed_sysroot"): + raise MediaSysrootError(f"sealed media manifest is missing: {manifest_path}") + return { + "runtime_profile": runtime_id, + "manifest": None, + **inspected, + } + + manifest = load_json(manifest_path, "sealed media manifest") + if manifest.get("runtime_profile") != runtime_id: + raise MediaSysrootError( + "sealed media runtime profile does not match the platform profile" + ) + if manifest.get("sources") != runtime.get("sources"): + raise MediaSysrootError("sealed media source identities do not match the lock") + if manifest.get("artifacts") != inspected["artifacts"]: + raise MediaSysrootError("sealed media artifact identities no longer match") + if manifest.get("links") != inspected["links"]: + raise MediaSysrootError("sealed media links no longer match") + return { + "runtime_profile": runtime_id, + "manifest": manifest_path, + **inspected, + } + + +def parse_source(value: str) -> tuple[str, str]: + name, separator, revision = value.partition("=") + if not separator or not name or not revision or "\n" in value: + raise argparse.ArgumentTypeError("source must be NAME=REVISION") + return name, revision + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + for command in ("seal", "verify"): + command_parser = subparsers.add_parser(command) + command_parser.add_argument("--platform-profile", required=True, type=Path) + command_parser.add_argument("--root", required=True, type=Path) + if command == "seal": + command_parser.add_argument( + "--source", action="append", type=parse_source, required=True + ) + args = parser.parse_args(argv) + try: + if args.command == "seal": + supplied_sources = dict(args.source) + if len(supplied_sources) != len(args.source): + raise MediaSysrootError("each --source name must be unique") + manifest_path = seal_sysroot( + args.platform_profile, args.root, supplied_sources + ) + print(f"SEALED {manifest_path}") + else: + result = verify_sysroot(args.platform_profile, args.root) + manifest = result["manifest"] or "unsealed-legacy-runtime" + print( + f"PASS runtime={result['runtime_profile']} " + f"manifest={manifest} root={args.root.resolve()}" + ) + except MediaSysrootError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/rknn/rknn_model_runner.cc b/tools/rknn/rknn_model_runner.cc index 0a414bc65..ac88a379a 100644 --- a/tools/rknn/rknn_model_runner.cc +++ b/tools/rknn/rknn_model_runner.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -68,9 +69,9 @@ void Check(int result, const std::string& action) { } } -rknn_core_mask ParseCoreMask(const std::string& value) { +std::optional ParseCoreMask(const std::string& value) { if (value == "auto") - return RKNN_NPU_CORE_AUTO; + return std::nullopt; if (value == "0") return RKNN_NPU_CORE_0; if (value == "1") @@ -182,7 +183,9 @@ int main(int argc, char** argv) { Check(rknn_init(context.Out(), const_cast(model.data()), static_cast(model.size()), 0, nullptr), "rknn_init"); - Check(rknn_set_core_mask(context.Get(), core_mask), "rknn_set_core_mask"); + if (core_mask) { + Check(rknn_set_core_mask(context.Get(), *core_mask), "rknn_set_core_mask"); + } rknn_sdk_version version{}; Check(rknn_query(context.Get(), RKNN_QUERY_SDK_VERSION, &version, sizeof(version)), diff --git a/tools/rknn/rknn_runtime_probe.cc b/tools/rknn/rknn_runtime_probe.cc index 1d648d9c9..6678ad3e3 100644 --- a/tools/rknn/rknn_runtime_probe.cc +++ b/tools/rknn/rknn_runtime_probe.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -55,9 +56,9 @@ void Check(int result, const std::string& action) { } } -rknn_core_mask ParseCoreMask(const std::string& value) { +std::optional ParseCoreMask(const std::string& value) { if (value == "auto") - return RKNN_NPU_CORE_AUTO; + return std::nullopt; if (value == "0") return RKNN_NPU_CORE_0; if (value == "1") @@ -101,7 +102,9 @@ int main(int argc, char** argv) { Check(rknn_init(context.Out(), const_cast(model.data()), static_cast(model.size()), 0, nullptr), "rknn_init"); - Check(rknn_set_core_mask(context.Get(), core_mask), "rknn_set_core_mask"); + if (core_mask) { + Check(rknn_set_core_mask(context.Get(), *core_mask), "rknn_set_core_mask"); + } rknn_sdk_version version{}; Check(rknn_query(context.Get(), RKNN_QUERY_SDK_VERSION, &version, sizeof(version)), diff --git a/tools/rknn/stage_platform_resources.py b/tools/rknn/stage_platform_resources.py new file mode 100644 index 000000000..20fc39d9e --- /dev/null +++ b/tools/rknn/stage_platform_resources.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +"""Stage target-specific RKNN resources from shared configs and explicit artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import shutil +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +MANIFEST_SCHEMA_VERSION = 2 + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def repository_path(raw: str | Path, field: str) -> Path: + path = (PROJECT_ROOT / raw).resolve() + try: + path.relative_to(PROJECT_ROOT) + except ValueError as error: + raise ValueError(f"{field} must stay in the repository") from error + return path + + +def parse_artifact(value: str) -> tuple[str, Path]: + name, separator, raw_path = value.partition("=") + if not separator or not name or not raw_path: + raise argparse.ArgumentTypeError("artifact must be MODEL_NAME=PATH") + path = Path(raw_path).expanduser().resolve() + if not path.is_file(): + raise argparse.ArgumentTypeError(f"artifact does not exist: {path}") + if path.suffix.lower() != ".rknn": + raise argparse.ArgumentTypeError(f"artifact must be an .rknn file: {path}") + return name, path + + +def replace_platform_tokens(value: Any, source_token: str, target_token: str) -> Any: + if isinstance(value, str): + return value.replace(source_token, target_token).replace( + source_token.lower(), target_token.lower() + ) + if isinstance(value, list): + return [replace_platform_tokens(item, source_token, target_token) for item in value] + if isinstance(value, dict): + return { + key: replace_platform_tokens(item, source_token, target_token) + for key, item in value.items() + } + return value + + +def model_templates(template_root: Path) -> dict[str, tuple[Path, dict[str, Any]]]: + templates: dict[str, tuple[Path, dict[str, Any]]] = {} + for config_path in sorted((template_root / "models").glob("*/config.json")): + config = json.loads(config_path.read_text(encoding="utf-8")) + code = str(config.get("algorithm_code", "")) + if not code: + raise ValueError(f"template model config has no algorithm_code: {config_path}") + if code in templates: + raise ValueError(f"duplicate template algorithm_code: {code}") + templates[code] = (config_path.parent, config) + if not templates: + raise ValueError(f"template resource has no model configs: {template_root}") + return templates + + +def algorithm_atomic_codes(document: dict[str, Any]) -> set[str]: + raw = document.get("atomicList", "[]") + try: + entries = json.loads(raw) if isinstance(raw, str) else raw + except json.JSONDecodeError as error: + raise ValueError("template algorithm atomicList is invalid JSON") from error + if not isinstance(entries, list): + raise ValueError("template algorithm atomicList must be an array") + return { + str(entry.get("atomicCode")) + for entry in entries + if isinstance(entry, dict) and entry.get("atomicCode") is not None + } + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def platform_context( + raw_profile_path: str | Path, raw_output_dir: str | Path | None = None +) -> tuple[Path, dict[str, Any], Path, Path, str]: + profile_path = repository_path(raw_profile_path, "--platform-profile") + if not profile_path.is_file(): + raise ValueError(f"platform profile does not exist: {profile_path}") + profile = json.loads(profile_path.read_text(encoding="utf-8")) + if profile.get("backend") != "rknn": + raise ValueError("platform profile backend must be rknn") + + packaging = profile.get("packaging", {}) + target_token = str(packaging.get("directory_token", "")) + if not re.fullmatch(r"[A-Z0-9]+", target_token): + raise ValueError("platform profile has no packaging.directory_token") + template_root = repository_path( + str(packaging.get("resource_template_directory", "")), + "packaging.resource_template_directory", + ) + default_output = str(packaging.get("resource_overlay_directory", "")) + output_dir = repository_path(raw_output_dir or default_output, "--output-dir") + return profile_path, profile, template_root, output_dir, target_token + + +def manifest_path(root: Path, document: dict[str, Any], field: str) -> Path: + raw_path = document.get("path") + if not isinstance(raw_path, str) or not raw_path: + raise ValueError(f"resource manifest has no {field}.path") + path = (root / raw_path).resolve() + try: + path.relative_to(root.resolve()) + except ValueError as error: + raise ValueError(f"resource manifest {field}.path escapes its root") from error + return path + + +def verify_file_record(root: Path, document: dict[str, Any], field: str) -> Path: + path = manifest_path(root, document, field) + if not path.is_file(): + raise ValueError(f"resource manifest {field} is missing: {path}") + expected_sha = document.get("sha256") + if not isinstance(expected_sha, str) or sha256(path) != expected_sha: + raise ValueError(f"resource manifest {field} hash mismatch: {path}") + return path + + +def verify_staged_resources( + raw_profile_path: str | Path, raw_output_dir: str | Path | None = None +) -> dict[str, Any]: + profile_path, profile, template_root, output_dir, target_token = platform_context( + raw_profile_path, raw_output_dir + ) + packaging = profile.get("packaging", {}) + requires_manifest = bool(packaging.get("resource_manifest_required", False)) + requires_manifest = requires_manifest or output_dir != template_root + if not requires_manifest: + if not (output_dir / "models").is_dir(): + raise ValueError(f"direct resource template has no models: {output_dir}") + return { + "status": "DIRECT_TEMPLATE", + "chip": profile["chip"], + "resource_root": output_dir.relative_to(PROJECT_ROOT).as_posix(), + } + + resource_manifest_path = output_dir / "resource-manifest.json" + if not resource_manifest_path.is_file(): + raise ValueError( + f"staged resource manifest is missing: {resource_manifest_path}; " + "regenerate the platform resource overlay" + ) + manifest = json.loads(resource_manifest_path.read_text(encoding="utf-8")) + if manifest.get("schema_version") != MANIFEST_SCHEMA_VERSION: + raise ValueError( + "staged resource manifest schema is stale: " + f"expected {MANIFEST_SCHEMA_VERSION}, got {manifest.get('schema_version')}" + ) + if manifest.get("chip") != profile.get("chip"): + raise ValueError("staged resource manifest chip does not match the platform profile") + if manifest.get("directory_token") != target_token: + raise ValueError("staged resource directory token does not match the platform profile") + + profile_record = manifest.get("platform_profile") + if not isinstance(profile_record, dict): + raise ValueError("staged resource manifest has no platform_profile record") + recorded_profile_path = verify_file_record( + PROJECT_ROOT, profile_record, "platform_profile" + ) + if recorded_profile_path != profile_path: + raise ValueError("staged resource manifest references a different platform profile") + + template_record = manifest.get("resource_template") + if not isinstance(template_record, dict): + raise ValueError("staged resource manifest has no resource_template record") + recorded_template_path = manifest_path( + PROJECT_ROOT, template_record, "resource_template" + ) + if recorded_template_path != template_root: + raise ValueError("staged resource manifest references a different resource template") + source_token = str(template_record.get("source_chip", "")) + if not source_token: + raise ValueError("staged resource manifest has no source chip token") + + model_records = manifest.get("models") + if not isinstance(model_records, list) or not model_records: + raise ValueError("staged resource manifest has no model records") + staged_codes: set[str] = set() + expected_model_files: set[str] = set() + seen_models: set[str] = set() + for index, record in enumerate(model_records): + field = f"models[{index}]" + if not isinstance(record, dict): + raise ValueError(f"resource manifest {field} must be an object") + model_name = record.get("model") + if not isinstance(model_name, str) or not model_name or model_name in seen_models: + raise ValueError(f"resource manifest {field} has an invalid or duplicate model") + seen_models.add(model_name) + code = str(record.get("algorithm_code", "")) + if not code: + raise ValueError(f"resource manifest {field} has no algorithm_code") + staged_codes.add(code) + + spec_record = record.get("spec") + source_record = record.get("source_template") + config_record = record.get("config") + artifact_record = record.get("artifact") + if not all( + isinstance(value, dict) + for value in (spec_record, source_record, config_record, artifact_record) + ): + raise ValueError(f"resource manifest {field} has incomplete file records") + spec_path = verify_file_record(PROJECT_ROOT, spec_record, f"{field}.spec") + spec = json.loads(spec_path.read_text(encoding="utf-8")) + if str(spec.get("packaging", {}).get("algorithm_code", "")) != code: + raise ValueError(f"resource manifest {field} model spec changed") + + source_config_path = verify_file_record( + PROJECT_ROOT, source_record, f"{field}.source_template" + ) + if source_config_path.parent.parent.parent != template_root: + raise ValueError(f"resource manifest {field} source template is outside the profile") + source_config = json.loads(source_config_path.read_text(encoding="utf-8")) + if str(source_config.get("algorithm_code", "")) != code: + raise ValueError(f"resource manifest {field} source template changed") + + target_config_path = verify_file_record( + output_dir, config_record, f"{field}.config" + ) + target_config = json.loads(target_config_path.read_text(encoding="utf-8")) + expected_config = replace_platform_tokens(source_config, source_token, target_token) + expected_config["chip_type"] = target_token + if target_config != expected_config: + raise ValueError( + "staged model config is stale relative to its source template: " + f"{target_config_path}" + ) + + target_model_path = verify_file_record( + output_dir, artifact_record, f"{field}.artifact" + ) + if artifact_record.get("size_bytes") != target_model_path.stat().st_size: + raise ValueError(f"resource manifest {field} artifact size mismatch") + if artifact_record.get("source_sha256") != artifact_record.get("sha256"): + raise ValueError(f"resource manifest {field} artifact differs from its source") + expected_model_files.update( + { + target_config_path.relative_to(output_dir).as_posix(), + target_model_path.relative_to(output_dir).as_posix(), + } + ) + + actual_model_files = { + path.relative_to(output_dir).as_posix() + for pattern in ("*/config.json", "*/model.rknn") + for path in (output_dir / "models").glob(pattern) + } + if actual_model_files != expected_model_files: + raise ValueError("staged model inventory does not match the resource manifest") + + algorithm_records = manifest.get("algorithms") + skipped_records = manifest.get("skipped_algorithms") + if not isinstance(algorithm_records, list) or not isinstance(skipped_records, list): + raise ValueError("staged resource manifest has invalid algorithm records") + expected_algorithm_files: set[str] = set() + recorded_sources: set[str] = set() + for index, record in enumerate(algorithm_records): + field = f"algorithms[{index}]" + if not isinstance(record, dict) or not isinstance(record.get("source"), dict): + raise ValueError(f"resource manifest {field} has no source record") + source_path = verify_file_record( + PROJECT_ROOT, record["source"], f"{field}.source" + ) + target_path = verify_file_record(output_dir, record, field) + source_document = json.loads(source_path.read_text(encoding="utf-8")) + target_document = json.loads(target_path.read_text(encoding="utf-8")) + if target_document != replace_platform_tokens( + source_document, source_token, target_token + ): + raise ValueError(f"staged algorithm config is stale: {target_path}") + required_codes = algorithm_atomic_codes(source_document) + if sorted(required_codes) != record.get("atomic_codes"): + raise ValueError(f"resource manifest {field} atomic codes changed") + if not required_codes.issubset(staged_codes): + raise ValueError(f"resource manifest {field} is missing required models") + recorded_sources.add(source_path.relative_to(PROJECT_ROOT).as_posix()) + expected_algorithm_files.add(target_path.relative_to(output_dir).as_posix()) + + for index, record in enumerate(skipped_records): + field = f"skipped_algorithms[{index}]" + if not isinstance(record, dict) or not isinstance(record.get("source"), dict): + raise ValueError(f"resource manifest {field} has no source record") + source_path = verify_file_record( + PROJECT_ROOT, record["source"], f"{field}.source" + ) + required_codes = algorithm_atomic_codes( + json.loads(source_path.read_text(encoding="utf-8")) + ) + missing_codes = sorted(required_codes - staged_codes) + if not missing_codes or missing_codes != record.get("missing_algorithm_codes"): + raise ValueError(f"resource manifest {field} selection changed") + recorded_sources.add(source_path.relative_to(PROJECT_ROOT).as_posix()) + + current_sources = { + path.relative_to(PROJECT_ROOT).as_posix() + for path in (template_root / "algorithm").glob("*.json") + } + if recorded_sources != current_sources: + raise ValueError("source algorithm inventory changed after resource staging") + actual_algorithm_files = { + path.relative_to(output_dir).as_posix() + for path in (output_dir / "algorithm").glob("*.json") + } + if actual_algorithm_files != expected_algorithm_files: + raise ValueError("staged algorithm inventory does not match the resource manifest") + + return { + "status": "PASS", + "chip": profile["chip"], + "manifest": resource_manifest_path.relative_to(PROJECT_ROOT).as_posix(), + "models": sorted(seen_models), + "algorithms": len(algorithm_records), + } + + +def stage_platform_resources( + raw_profile_path: str | Path, + artifacts: list[tuple[str, Path]], + raw_output_dir: str | Path | None = None, + force: bool = False, +) -> dict[str, Any]: + profile_path, profile, template_root, output_dir, target_token = platform_context( + raw_profile_path, raw_output_dir + ) + allowed_output_root = (PROJECT_ROOT / "output" / "platform-artifacts").resolve() + try: + output_dir.relative_to(allowed_output_root) + except ValueError as error: + raise ValueError( + "generated platform resources must stay under output/platform-artifacts" + ) from error + if output_dir == template_root or template_root in output_dir.parents: + raise ValueError("generated platform resources must not overwrite the resource template") + if output_dir.exists(): + if not force: + raise ValueError(f"output already exists: {output_dir}; pass --force to replace it") + shutil.rmtree(output_dir) + + templates = model_templates(template_root) + source_tokens = { + str(config.get("chip_type", "")) + for _, config in templates.values() + if config.get("chip_type") + } + if len(source_tokens) != 1: + raise ValueError("template model configs must share one chip_type") + source_token = next(iter(source_tokens)) + + supplied = dict(artifacts) + if not supplied: + raise ValueError("at least one --artifact is required when staging resources") + if len(supplied) != len(artifacts): + raise ValueError("each --artifact model name must be unique") + staged_codes: set[str] = set() + records: list[dict[str, Any]] = [] + for model_name, artifact_path in sorted(supplied.items()): + spec_path = PROJECT_ROOT / "config" / "rknn" / "models" / f"{model_name}.json" + if not spec_path.is_file(): + raise ValueError(f"model spec does not exist: {spec_path}") + spec = json.loads(spec_path.read_text(encoding="utf-8")) + code = str(spec.get("packaging", {}).get("algorithm_code", "")) + if code not in templates: + raise ValueError( + f"no resource template for {model_name} algorithm_code={code or 'missing'}" + ) + source_dir, source_config = templates[code] + source_config_path = source_dir / "config.json" + target_dir_name = source_dir.name.replace(source_token, target_token) + target_dir = output_dir / "models" / target_dir_name + target_config = replace_platform_tokens(source_config, source_token, target_token) + target_config["chip_type"] = target_token + write_json(target_dir / "config.json", target_config) + target_model = target_dir / "model.rknn" + target_model.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(artifact_path, target_model) + staged_codes.add(code) + records.append( + { + "model": model_name, + "algorithm_code": code, + "spec": { + "path": spec_path.relative_to(PROJECT_ROOT).as_posix(), + "sha256": sha256(spec_path), + }, + "source_template": { + "path": source_config_path.relative_to(PROJECT_ROOT).as_posix(), + "sha256": sha256(source_config_path), + }, + "config": { + "path": (target_dir / "config.json").relative_to(output_dir).as_posix(), + "sha256": sha256(target_dir / "config.json"), + }, + "artifact": { + "source_sha256": sha256(artifact_path), + "path": target_model.relative_to(output_dir).as_posix(), + "sha256": sha256(target_model), + "size_bytes": target_model.stat().st_size, + }, + } + ) + + algorithm_records = [] + skipped_algorithms = [] + for source_path in sorted((template_root / "algorithm").glob("*.json")): + source_document = json.loads(source_path.read_text(encoding="utf-8")) + source_record = { + "path": source_path.relative_to(PROJECT_ROOT).as_posix(), + "sha256": sha256(source_path), + } + required_codes = algorithm_atomic_codes(source_document) + if not required_codes.issubset(staged_codes): + skipped_algorithms.append( + { + "source": source_record, + "missing_algorithm_codes": sorted(required_codes - staged_codes), + } + ) + continue + target_document = replace_platform_tokens(source_document, source_token, target_token) + target_name = source_path.name.replace(source_token, target_token) + target_path = output_dir / "algorithm" / target_name + write_json(target_path, target_document) + algorithm_records.append( + { + "source": source_record, + "path": target_path.relative_to(output_dir).as_posix(), + "sha256": sha256(target_path), + "atomic_codes": sorted(required_codes), + } + ) + + manifest = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "created_at": datetime.now(timezone.utc).isoformat(), + "chip": profile["chip"], + "directory_token": target_token, + "platform_profile": { + "path": profile_path.relative_to(PROJECT_ROOT).as_posix(), + "sha256": sha256(profile_path), + }, + "resource_template": { + "path": template_root.relative_to(PROJECT_ROOT).as_posix(), + "source_chip": source_token, + }, + "models": records, + "algorithms": algorithm_records, + "skipped_algorithms": skipped_algorithms, + } + write_json(output_dir / "resource-manifest.json", manifest) + verify_staged_resources(profile_path, output_dir) + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--platform-profile", required=True) + parser.add_argument("--artifact", action="append", type=parse_artifact, default=[]) + parser.add_argument("--output-dir") + parser.add_argument("--force", action="store_true") + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + try: + if args.verify: + if args.artifact or args.force: + parser.error("--verify cannot be combined with --artifact or --force") + result = verify_staged_resources(args.platform_profile, args.output_dir) + else: + result = stage_platform_resources( + args.platform_profile, args.artifact, args.output_dir, args.force + ) + except (OSError, ValueError, json.JSONDecodeError) as error: + parser.error(str(error)) + print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/scenario-bench/README.md b/tools/scenario-bench/README.md index 35cf4ec31..b412e7cc8 100644 --- a/tools/scenario-bench/README.md +++ b/tools/scenario-bench/README.md @@ -76,11 +76,15 @@ node src/cli.js checkpoint \ --max-gap-sec 120 \ --min-fps-ratio 0.9 \ --expected-preview-streams 4 \ - --expected-decoder-backend rockchip-copy-out \ - --expected-encoder-backend rockchip-copy-first + --expected-decoder-backend rockchip-mpp-rga \ + --expected-encoder-backend rockchip-mpp-rga \ + --min-rga-bound-native-int8-frames 1 \ + --min-rknn-mpp-dmabuf-frames 1 \ + --min-rknn-rga-crop-dmabuf-frames 1 \ + --max-rga-bound-requantize-avg-ms 2 ``` -输出结论只有三种：`IN_PROGRESS` 表示时长尚未达到且其余门禁正常（命令退出码 3），`FAIL` 表示连续性、负载、资源或原生媒体链路存在失败（退出码 1），`PASS` 仅在时长达到且全部必需检查通过时产生（退出码 0）。审计覆盖采样间断/新鲜度、固定路数、逐通道 FPS 与丢帧、CPU/内存/磁盘、内存池增长、MPP/RGA/RKNN 失败及计数器回退、延迟 Copy-out 记账、OSD/编码/发布帧流和 SRS 发布流；输入和候选身份文件的 SHA-256 会写入结果。 +输出结论只有三种：`IN_PROGRESS` 表示时长尚未达到且其余门禁正常（命令退出码 3），`FAIL` 表示连续性、负载、资源或原生媒体链路存在失败（退出码 1），`PASS` 仅在时长达到且全部必需检查通过时产生（命令退出码 0）。`gate-hours` 从首次观测到目标路数、活动路数和唯一通道数均达到配置负载时开始累计；登录、建连和部分路数爬坡只计入 `runAgeSec`，不计入验收使用的 `fullLoadCoverageSec`。审计覆盖采样间断/新鲜度、固定路数、逐通道 FPS 与丢帧、CPU/内存/磁盘、内存池增长、MPP/RGA/RKNN 失败及计数器回退、延迟 Copy-out 记账、OSD/编码/发布帧流和 SRS 发布流；针对原生 RKNN 路径还可要求 MPP DMA-BUF 输入、分类裁剪 DMA-BUF、全部 forward 的 INT8 直绑覆盖和必要符号位变换的平均耗时。输入和候选身份文件的 SHA-256 会写入结果。 ## 创建场景包 diff --git a/tools/scenario-bench/src/cli.js b/tools/scenario-bench/src/cli.js index 7923fc9ef..f78f4698b 100644 --- a/tools/scenario-bench/src/cli.js +++ b/tools/scenario-bench/src/cli.js @@ -109,7 +109,7 @@ init-scenario options: checkpoint options: --input Running metrics.partial.json or completed metrics.json --output Atomic JSON checkpoint output path - --gate-hours Required continuous runtime, e.g. 12 + --gate-hours Required observed full-load runtime; excludes login/ramp, e.g. 12 --identity Optional immutable candidate identity key=value file --source-label Canonical remote/source path recorded in evidence --identity-label Canonical identity path recorded in evidence @@ -121,6 +121,10 @@ checkpoint options: --expected-decoder-backend Required decoder backend identifier --expected-encoder-backend Required encoder backend identifier --min-rga-bound-uint8-frames Required fused RGA-to-RKNN UINT8 frames + --min-rga-bound-native-int8-frames Required native INT8 bound frames with full RKNN accounting + --min-rknn-mpp-dmabuf-frames Required RKNN frames sourced from MPP DMA-BUF + --min-rknn-rga-crop-dmabuf-frames Required classifier crops sourced from DMA-BUF + --max-rga-bound-requantize-avg-ms Maximum average native U8-to-INT8 transform latency `); } @@ -751,6 +755,10 @@ async function runCheckpoint(args) { maxPoolGrowthBytes: optionalNumber(args, 'max-pool-growth-bytes'), poolGrowthWarmupSec: optionalNumber(args, 'pool-growth-warmup-sec'), minRgaBoundUint8Frames: optionalNumber(args, 'min-rga-bound-uint8-frames'), + minRgaBoundNativeInt8Frames: optionalNumber(args, 'min-rga-bound-native-int8-frames'), + minRknnMppDmaBufFrames: optionalNumber(args, 'min-rknn-mpp-dmabuf-frames'), + minRknnRgaCropDmaBufFrames: optionalNumber(args, 'min-rknn-rga-crop-dmabuf-frames'), + maxRgaBoundRequantizeAvgMs: optionalNumber(args, 'max-rga-bound-requantize-avg-ms'), expectedPreviewStreams: optionalNumber(args, 'expected-preview-streams'), expectedDecoderBackend: args['expected-decoder-backend'], expectedEncoderBackend: args['expected-encoder-backend'], diff --git a/tools/scenario-bench/src/longrun-auditor.js b/tools/scenario-bench/src/longrun-auditor.js index a25913399..c6ae740d5 100644 --- a/tools/scenario-bench/src/longrun-auditor.js +++ b/tools/scenario-bench/src/longrun-auditor.js @@ -7,6 +7,10 @@ const MIB = 1024 * 1024; const REQUIRED_FAILURE_COUNTERS = [ 'graphForwardFailures', 'mppCopyOutFailures', + 'mppRgaCopyOutFailures', + 'mppCpuCopyOutFallbacks', + 'mppRgaCopyInFailures', + 'mppCpuCopyInFallbacks', 'mppDecodeFailures', 'mppDecodeFallbacks', 'mppEncodeFailures', @@ -14,9 +18,36 @@ const REQUIRED_FAILURE_COUNTERS = [ 'resultParseFailures', 'rgaFailures', 'rknnForwardFailures', + 'rknnBoundInputBindFailures', + 'rknnBoundInputCopyFailures', + 'rknnBoundInputSyncFailures', + 'rknnCpuResizeFallbackCalls', + 'rknnCpuCropResizeFallbackCalls', + 'rknnCpuNormalizeFallbackCalls', + 'rknnDetectorForwardFailures', + 'rknnInputCompatibilityFallbacks', + 'rknnMppDmaBufImportFailures', + 'rknnOutputCompatibilityFallbacks', 'rknnRgaFailures', + 'rknnRgaCropResizeFailures', + 'rknnRgaCropHostFallbacks', + 'rknnRgaBoundInputBindFailures', 'rknnRgaBoundInputImportFailures', 'rknnRgaBoundRequantizeFailures', + 'rknnMppDmaBufFallbacks', + 'rknnYolov8DirectCandidateFailures', +]; + +const FAILURE_COUNTER_SUFFIX = /(Failures|Fallbacks|FallbackCalls)$/; + +const NATIVE_BOUND_FORBIDDEN_COUNTERS = [ + 'rknnBoundInputCopyCalls', + 'rknnBoundInputCopyBytes', + 'rknnNativeInputMapCalls', + 'rknnFloatInputs', + 'rknnUint8ContractInputs', + 'rknnInputsSetCalls', + 'rknnDetectorInputsSetCalls', ]; const MONOTONIC_COUNTERS = [ @@ -24,6 +55,8 @@ const MONOTONIC_COUNTERS = [ 'colorConvertFrames', 'graphForwardFrames', 'mppCopyOutFrames', + 'mppRgaCopyOutFrames', + 'mppRgaCopyInFrames', 'mppDecodedFrames', 'mppEarlyDroppedFrames', 'mppEncodedFrames', @@ -32,8 +65,17 @@ const MONOTONIC_COUNTERS = [ 'resultParseFrames', 'rgaFrames', 'rknnForwards', + 'rknnRgaCropResizeCalls', + 'rknnRgaCropDmaBufFrames', + 'rknnRgaCropHostFallbacks', 'rknnRgaBoundInputFrames', 'rknnRgaBoundUint8Frames', + 'rknnRgaBoundNativeInt8Frames', + 'rknnRgaBoundRequantizeCalls', + 'rknnRgaBoundRequantizeMs', + 'rknnMppDmaBufFrames', + 'rknnBoundInputSyncCalls', + 'rknnMppDmaBufImportCalls', ]; const IDENTITY_ALLOWLIST = new Set([ @@ -89,6 +131,9 @@ function counterSummary(samples, key) { samples: values.length, first: values[0] ?? null, last: values.at(-1) ?? null, + min: values.length ? Math.min(...values) : null, + max: values.length ? Math.max(...values) : null, + nonZeroSamples: values.filter((value) => value !== 0).length, delta: values.length ? values.at(-1) - values[0] : null, decreases: values.slice(1).reduce( (count, value, index) => count + (value < values[index] ? 1 : 0), @@ -149,6 +194,18 @@ export function auditLongRun(runResult, options = {}) { const minRgaBoundUint8Frames = options.minRgaBoundUint8Frames == null ? null : Number(options.minRgaBoundUint8Frames); + const minRgaBoundNativeInt8Frames = options.minRgaBoundNativeInt8Frames == null + ? null + : Number(options.minRgaBoundNativeInt8Frames); + const minRknnMppDmaBufFrames = options.minRknnMppDmaBufFrames == null + ? null + : Number(options.minRknnMppDmaBufFrames); + const minRknnRgaCropDmaBufFrames = options.minRknnRgaCropDmaBufFrames == null + ? null + : Number(options.minRknnRgaCropDmaBufFrames); + const maxRgaBoundRequantizeAvgMs = options.maxRgaBoundRequantizeAvgMs == null + ? null + : Number(options.maxRgaBoundRequantizeAvgMs); const nowMs = Number(options.nowMs ?? Date.now()); const expectedPreviewStreams = options.expectedPreviewStreams == null ? null @@ -177,6 +234,19 @@ export function auditLongRun(runResult, options = {}) { && (!Number.isInteger(minRgaBoundUint8Frames) || minRgaBoundUint8Frames < 0)) { throw new Error('minRgaBoundUint8Frames must be a non-negative integer'); } + for (const [name, value] of Object.entries({ + minRgaBoundNativeInt8Frames, + minRknnMppDmaBufFrames, + minRknnRgaCropDmaBufFrames, + })) { + if (value != null && (!Number.isInteger(value) || value < 0)) { + throw new Error(`${name} must be a non-negative integer`); + } + } + if (maxRgaBoundRequantizeAvgMs != null + && (!Number.isFinite(maxRgaBoundRequantizeAvgMs) || maxRgaBoundRequantizeAvgMs < 0)) { + throw new Error('maxRgaBoundRequantizeAvgMs must be a non-negative number'); + } const checks = []; const add = (...args) => checks.push(check(...args)); @@ -197,6 +267,22 @@ export function auditLongRun(runResult, options = {}) { const runAgeSec = Number.isFinite(startedMs) && effectiveEndMs != null ? (effectiveEndMs - startedMs) / 1000 : null; + const fullLoadSamples = samples.filter((sample) => { + const targetChannels = Number(sample?.targetChannels); + const activeChannels = Number(sample?.activeChannels); + const uniqueChannels = new Set((sample?.channels ?? []).map((channel) => channel.channelId)); + return Number.isFinite(Number(sample?.ts)) + && Number.isInteger(targetChannels) + && targetChannels > 0 + && activeChannels === targetChannels + && uniqueChannels.size === targetChannels; + }); + const firstFullLoadTs = Number(fullLoadSamples[0]?.ts); + const fullLoadCoverageSec = Number.isFinite(firstFullLoadTs) + && effectiveEndMs != null + && effectiveEndMs >= firstFullLoadTs + ? (effectiveEndMs - firstFullLoadTs) / 1000 + : null; const sampleSpanSec = firstTs != null && lastTs != null ? (lastTs - firstTs) / 1000 : null; const firstSampleDelaySec = Number.isFinite(startedMs) && firstTs != null ? (firstTs - startedMs) / 1000 : null; const finalSampleDelaySec = runResult.status === 'completed' @@ -256,12 +342,13 @@ export function auditLongRun(runResult, options = {}) { ); } - const durationReached = runAgeSec != null && runAgeSec >= requiredDurationSec; + const durationReached = fullLoadCoverageSec != null + && fullLoadCoverageSec >= requiredDurationSec; add( 'gate.duration', durationReached ? 'PASS' : (runResult.status === 'running' ? 'IN_PROGRESS' : 'FAIL'), - round(runAgeSec), - `>= ${requiredDurationSec}s (${gateHours}h)`, + round(fullLoadCoverageSec), + `>= ${requiredDurationSec}s (${gateHours}h) of observed full-load coverage`, ); const stepIndexes = [...new Set(holdSamples.map((sample) => sample.stepIndex))]; @@ -457,21 +544,36 @@ export function auditLongRun(runResult, options = {}) { 'accelerator telemetry present in every hold sample', ); - const counterKeys = [...new Set([...MONOTONIC_COUNTERS, ...REQUIRED_FAILURE_COUNTERS])]; + const dynamicFailureCounterKeys = [...new Set(accelerators.flatMap((accelerator) => ( + accelerator + ? Object.keys(accelerator).filter((key) => FAILURE_COUNTER_SUFFIX.test(key)) + : [] + )))].sort(); + const failureCounterKeys = [...new Set([ + ...REQUIRED_FAILURE_COUNTERS, + ...dynamicFailureCounterKeys, + ])]; + const gatedForbiddenCounters = minRgaBoundNativeInt8Frames == null + ? [] + : NATIVE_BOUND_FORBIDDEN_COUNTERS; + const counterKeys = [...new Set([ + ...MONOTONIC_COUNTERS, + ...failureCounterKeys, + ...gatedForbiddenCounters, + ])]; const counters = Object.fromEntries( counterKeys.map((key) => [key, counterSummary(holdSamples, key)]), ); const missingCounters = counterKeys.filter((key) => counters[key].samples !== holdSamples.length); const resetCounters = MONOTONIC_COUNTERS.filter((key) => counters[key].decreases > 0); - const failureDeltas = Object.fromEntries( - REQUIRED_FAILURE_COUNTERS.map((key) => [key, counters[key].delta]), + const nonZeroFailures = failureCounterKeys.filter( + (key) => counters[key].nonZeroSamples > 0, ); - const nonZeroFailures = Object.entries(failureDeltas).filter(([, delta]) => delta !== 0); add( 'native.countersPresent', missingCounters.length === 0 ? 'PASS' : 'FAIL', missingCounters, - 'all required native-media counters present in every hold sample', + 'all explicit, dynamically discovered, and gated native-media counters present in every hold sample', ); add( 'native.counterContinuity', @@ -482,8 +584,8 @@ export function auditLongRun(runResult, options = {}) { add( 'native.failures', nonZeroFailures.length === 0 ? 'PASS' : 'FAIL', - Object.fromEntries(nonZeroFailures), - 'all failure/fallback counter deltas equal zero', + Object.fromEntries(nonZeroFailures.map((key) => [key, counters[key].max])), + 'every failure/fallback counter equals zero in every hold sample', ); if (minRgaBoundUint8Frames != null) { const fusedFrames = counters.rknnRgaBoundUint8Frames?.delta; @@ -494,6 +596,100 @@ export function auditLongRun(runResult, options = {}) { `>= ${minRgaBoundUint8Frames} fused UINT8 bound-input frames`, ); } + if (minRgaBoundNativeInt8Frames != null) { + const missingForbiddenCounters = NATIVE_BOUND_FORBIDDEN_COUNTERS.filter( + (key) => counters[key].samples !== holdSamples.length, + ); + const activeForbiddenCounters = NATIVE_BOUND_FORBIDDEN_COUNTERS.filter( + (key) => counters[key].nonZeroSamples > 0, + ); + add( + 'native.legacyInputPaths', + missingForbiddenCounters.length === 0 && activeForbiddenCounters.length === 0 + ? 'PASS' + : 'FAIL', + { + missing: missingForbiddenCounters, + nonZero: Object.fromEntries( + activeForbiddenCounters.map((key) => [key, counters[key].max]), + ), + }, + 'legacy host-copy, mapped-input, compatibility-input, and rknn_inputs_set paths stay present and zero in every hold sample', + ); + const forwards = counters.rknnForwards?.delta; + const boundFrames = counters.rknnRgaBoundInputFrames?.delta; + const fusedFrames = counters.rknnRgaBoundUint8Frames?.delta; + const nativeFrames = counters.rknnRgaBoundNativeInt8Frames?.delta; + const requantizeCalls = counters.rknnRgaBoundRequantizeCalls?.delta; + const accountingTolerance = Number.isFinite(forwards) + ? Math.max(2, Math.ceil(forwards * 0.0001)) + : null; + const forwardCoverageError = [forwards, boundFrames].every(Number.isFinite) + ? Math.abs(forwards - boundFrames) + : null; + const nativeCoverageError = [boundFrames, nativeFrames].every(Number.isFinite) + ? Math.abs(boundFrames - nativeFrames) + : null; + const requantizeCoverageError = [nativeFrames, requantizeCalls].every(Number.isFinite) + ? Math.abs(nativeFrames - requantizeCalls) + : null; + add( + 'native.rgaBoundNativeInt8', + Number.isFinite(nativeFrames) + && nativeFrames >= minRgaBoundNativeInt8Frames + && fusedFrames === 0 + && forwardCoverageError != null + && forwardCoverageError <= accountingTolerance + && nativeCoverageError != null + && nativeCoverageError <= accountingTolerance + && requantizeCoverageError != null + && requantizeCoverageError <= accountingTolerance + ? 'PASS' + : 'FAIL', + { + forwards, + boundFrames, + fusedFrames, + nativeFrames, + requantizeCalls, + forwardCoverageError, + nativeCoverageError, + requantizeCoverageError, + }, + `>= ${minRgaBoundNativeInt8Frames} native INT8 frames, no fused UINT8 frames, and complete forward/bind/requantize accounting (tolerance ${accountingTolerance})`, + ); + } + if (minRknnMppDmaBufFrames != null) { + const frames = counters.rknnMppDmaBufFrames?.delta; + add( + 'native.rknnMppDmaBuf', + Number.isFinite(frames) && frames >= minRknnMppDmaBufFrames ? 'PASS' : 'FAIL', + frames ?? null, + `>= ${minRknnMppDmaBufFrames} RKNN frames sourced from MPP DMA-BUF`, + ); + } + if (minRknnRgaCropDmaBufFrames != null) { + const frames = counters.rknnRgaCropDmaBufFrames?.delta; + add( + 'native.rknnRgaCropDmaBuf', + Number.isFinite(frames) && frames >= minRknnRgaCropDmaBufFrames ? 'PASS' : 'FAIL', + frames ?? null, + `>= ${minRknnRgaCropDmaBufFrames} classifier crops sourced from DMA-BUF`, + ); + } + if (maxRgaBoundRequantizeAvgMs != null) { + const calls = counters.rknnRgaBoundRequantizeCalls?.delta; + const elapsedMs = counters.rknnRgaBoundRequantizeMs?.delta; + const averageMs = Number.isFinite(calls) && calls > 0 && Number.isFinite(elapsedMs) + ? elapsedMs / calls + : null; + add( + 'native.rgaBoundRequantizeLatency', + averageMs != null && averageMs <= maxRgaBoundRequantizeAvgMs ? 'PASS' : 'FAIL', + round(averageMs, 6), + `<= ${maxRgaBoundRequantizeAvgMs} ms average native U8-to-INT8 sign-bit transform`, + ); + } const backendValues = (key) => [...new Set( accelerators.map((value) => value?.[key]).filter(Boolean), @@ -536,7 +732,19 @@ export function auditLongRun(runResult, options = {}) { { decoded, copied, earlyDropped, error: copyAccountingError }, `decoded ~= copied + earlyDropped (tolerance ${copyAccountingTolerance})`, ); - if (options.expectedDecoderBackend === 'rockchip-copy-out') { + const rgaCopiedOut = counters.mppRgaCopyOutFrames?.delta; + const cpuCopyOutFallbacks = counters.mppCpuCopyOutFallbacks?.delta; + const rgaCopyOutError = [copied, rgaCopiedOut].every(Number.isFinite) + ? Math.abs(copied - rgaCopiedOut) + : null; + add( + 'native.rgaCopyOut', + rgaCopyOutError != null && rgaCopyOutError <= copyAccountingTolerance + && cpuCopyOutFallbacks === 0 ? 'PASS' : 'FAIL', + { copied, rgaCopiedOut, cpuCopyOutFallbacks, error: rgaCopyOutError }, + `all materialized frames use RGA (tolerance ${copyAccountingTolerance})`, + ); + if (options.expectedDecoderBackend?.startsWith('rockchip-')) { add( 'native.earlyDropActive', earlyDropped > 0 ? 'PASS' : 'FAIL', @@ -546,6 +754,21 @@ export function auditLongRun(runResult, options = {}) { } const encoded = counters.mppEncodedFrames?.delta; + const rgaCopiedIn = counters.mppRgaCopyInFrames?.delta; + const cpuCopyInFallbacks = counters.mppCpuCopyInFallbacks?.delta; + const copyInAccountingTolerance = Number.isFinite(encoded) + ? Math.max(8, Math.ceil(encoded * 0.001)) + : null; + const copyInAccountingError = [encoded, rgaCopiedIn].every(Number.isFinite) + ? Math.abs(encoded - rgaCopiedIn) + : null; + add( + 'native.rgaCopyIn', + copyInAccountingError != null && copyInAccountingError <= copyInAccountingTolerance + && cpuCopyInFallbacks === 0 ? 'PASS' : 'FAIL', + { encoded, rgaCopiedIn, cpuCopyInFallbacks, error: copyInAccountingError }, + `all encoded frames use RGA copy-in (tolerance ${copyInAccountingTolerance})`, + ); const osd = counters.osdFrames?.delta; const published = counters.publishedFrames?.delta; const publishRatio = Number.isFinite(encoded) && encoded > 0 && Number.isFinite(published) @@ -618,6 +841,10 @@ export function auditLongRun(runResult, options = {}) { requiredDurationSec, reached: durationReached, runAgeSec: round(runAgeSec), + fullLoadCoverageSec: round(fullLoadCoverageSec), + firstFullLoadAt: Number.isFinite(firstFullLoadTs) + ? new Date(firstFullLoadTs).toISOString() + : null, sampleSpanSec: round(sampleSpanSec), firstSampleDelaySec: round(firstSampleDelaySec), finalSampleDelaySec: round(finalSampleDelaySec), @@ -655,6 +882,9 @@ export function auditLongRun(runResult, options = {}) { nativeMedia: { decoderBackends, encoderBackends, + failureCounterKeys, + dynamicFailureCounterKeys, + nativeBoundForbiddenCounters: gatedForbiddenCounters, counters, copyAccountingError, copyAccountingTolerance, diff --git a/tools/scenario-bench/src/report-writer.js b/tools/scenario-bench/src/report-writer.js index 9d1812fb7..1735cf187 100644 --- a/tools/scenario-bench/src/report-writer.js +++ b/tools/scenario-bench/src/report-writer.js @@ -270,8 +270,8 @@ export class ReportWriter { ${metric(m.rknnRunAvgMs)}/${metric(m.rknnOutputsGetAvgMs)}/${metric(m.rknnOutputsReleaseAvgMs)}/${metric(m.rknnOutputTransformAvgMs)} ${metric(m.rknnForwardAvgMs)}/${m.rknnForwardFailures ?? '-'} ${metric(m.rknnDetectorForwardAvgMs)}/${metric(m.rknnDetectorMutexWaitAvgMs)}/${m.rknnDetectorForwardFailures ?? '-'} - ${metric(m.rknnRgaFillAvgMs)}/${metric(m.rknnRgaResizeColorAvgMs)}/${metric(m.rknnNativeInputMapAvgMs)}/${m.rknnPreprocessFastHits ?? '-'}/${m.rknnRgaFailures ?? '-'} - ${m.rknnCpuResizeFallbacks ?? '-'}/${m.rknnCpuNormalizeFallbacks ?? '-'}/${m.rknnInputCompatibilityFallbacks ?? '-'} + ${metric(m.rknnRgaFillAvgMs)}/${metric(m.rknnRgaResizeColorAvgMs)}/${metric(m.rknnRgaCropResizeAvgMs)}/${m.rknnRgaCropResizeCalls ?? '-'}/${m.rknnRgaCropDmaBufFrames ?? '-'}/${m.rknnRgaCropHostFallbacks ?? '-'}/${metric(m.rknnNativeInputMapAvgMs)}/${m.rknnPreprocessFastHits ?? '-'}/${m.rknnRgaFailures ?? '-'}/${m.rknnRgaCropResizeFailures ?? '-'} + ${m.rknnCpuResizeFallbacks ?? '-'}/${m.rknnCpuCropResizeFallbacks ?? '-'}/${m.rknnCpuNormalizeFallbacks ?? '-'}/${m.rknnInputCompatibilityFallbacks ?? '-'} ${m.rknnBoundInputBindAttempts ?? '-'}/${m.rknnBoundInputBindFailures ?? '-'}/${m.rknnBoundInputFrames ?? '-'}/${metric(m.rknnBoundInputCopyAvgMs)}/${metric(m.rknnBoundInputSyncAvgMs)}/${formatMib(m.rknnBoundInputCopyAvgBytes)}/${m.rknnBoundInputCopyFailures ?? '-'}/${m.rknnBoundInputSyncFailures ?? '-'} ${m.rknnRgaBoundInputBindAttempts ?? '-'}/${m.rknnRgaBoundInputBindFailures ?? '-'}/${m.rknnRgaBoundInputImportCalls ?? '-'}/${metric(m.rknnRgaBoundInputImportAvgMs)}/${m.rknnRgaBoundInputImportFailures ?? '-'}/${m.rknnRgaBoundInputFrames ?? '-'}/${m.rknnRgaBoundUint8Frames ?? '-'}/${m.rknnRgaBoundNativeInt8Frames ?? '-'}/${m.rknnRgaBoundRequantizeCalls ?? '-'}/${metric(m.rknnRgaBoundRequantizeAvgMs)}/${m.rknnRgaBoundRequantizeFailures ?? '-'}/${m.rknnRgaBoundInputNormalizeBypasses ?? '-'} ${m.rknnMppDmaBufImportCalls ?? '-'}/${metric(m.rknnMppDmaBufImportAvgMs)}/${m.rknnMppDmaBufImportFailures ?? '-'}/${m.rknnMppDmaBufFrames ?? '-'}/${m.rknnMppDmaBufFallbacks ?? '-'}/${formatMib(m.rknnMppDmaBufSourceAvgBytes)} @@ -280,9 +280,9 @@ export class ReportWriter { ${m.rknnYolov8DirectCandidateCalls ?? '-'}/${m.rknnYolov8DirectCandidateFailures ?? '-'}/${metric(m.rknnYolov8DirectAvgPointsScanned)}/${metric(m.rknnYolov8DirectAvgPointsDecoded)}/${metric(m.rknnYolov8ScoreSumAvgPointsRejected)}/${formatMib(m.rknnYolov8LogicalFloatBytesAvoided)} ${metric(m.yolov8PostprocessAvgMs)}/${metric(m.yolov8NmsAvgMs)} ${metric(m.rgaAvgMs)}/${m.rgaFailures ?? '-'} - ${metric(m.mppEncodeAvgMs)}/${m.mppEncodeFailures ?? '-'} + ${metric(m.mppEncodeAvgMs)}/${m.mppEncodeFailures ?? '-'}/${m.mppRgaCopyInFrames ?? '-'}/${m.mppRgaCopyInFailures ?? '-'}/${m.mppCpuCopyInFallbacks ?? '-'} ${metric(m.mppDecodeAvgMs)}/${m.mppDecodeFailures ?? '-'}/${m.mppDecodeFallbacks ?? '-'} - ${metric(m.mppCopyOutAvgMs)}/${m.mppDecodedFrames ?? '-'}/${m.mppCopyOutFrames ?? '-'}/${m.mppEarlyDroppedFrames ?? '-'}/${m.mppCopyOutFailures ?? '-'} + ${metric(m.mppCopyOutAvgMs)}/${m.mppDecodedFrames ?? '-'}/${m.mppCopyOutFrames ?? '-'}/${m.mppRgaCopyOutFrames ?? '-'}/${m.mppCpuCopyOutFallbacks ?? '-'}/${m.mppEarlyDroppedFrames ?? '-'}/${m.mppCopyOutFailures ?? '-'} ${metric(m.osdAvgMs)} ${metric(m.publishAvgMs)} ${metric(m.firstFrameAvgMs)}/${metric(m.firstFrameMaxMs)} @@ -326,7 +326,7 @@ ${bottleneckBanner}

媒体与预览分阶段指标

- + ${mediaRows}
路数Preprocess msInfer msPostprocess ms颜色/Blob msGraph/Parse msRKNN准备/送入 msRKNN执行/取回/释放/转换 msRKNN总计/失败Detector总计/等待/失败Fast Fill/Resize/Map/命中/失败Fallback Resize/Normalize/Compat绑定输入 Bind/失败/帧/Copy ms/Sync ms/CopyMiB/Copy失败/Sync失败RGA直绑 Bind/失败/Import次数/ms/失败/帧/UINT8融合帧/INT8回退帧/Requant次数/ms/失败/Normalize绕过MPP DMA-BUF Import/ms/失败/帧/回退/源MiB输出 Native/Float/Compat/NativeMiB/FloatMiB量化 DFL/Class ms直接候选 调用/失败/扫描/解码/Sum早筛/省略MiBYOLO Post/NMSRGA/失败MPP编码/失败MPP解码/失败/回退Copy-out ms/解码/复制/早丢/失败OSD msPublish ms首帧平均/进程最大ms预览流/发布器峰值原始/算法预览峰值SRS流/客户端峰值启动/停止/失败增量
路数Preprocess msInfer msPostprocess ms颜色/Blob msGraph/Parse msRKNN准备/送入 msRKNN执行/取回/释放/转换 msRKNN总计/失败Detector总计/等待/失败Fast Fill/Resize/Crop次数/CropDMA/CropHost/Map/命中/失败Fallback Resize/Crop/Normalize/Compat绑定输入 Bind/失败/帧/Copy ms/Sync ms/CopyMiB/Copy失败/Sync失败RGA直绑 Bind/失败/Import次数/ms/失败/帧/UINT8融合帧/INT8回退帧/Requant次数/ms/失败/Normalize绕过MPP DMA-BUF Import/ms/失败/帧/回退/源MiB输出 Native/Float/Compat/NativeMiB/FloatMiB量化 DFL/Class ms直接候选 调用/失败/扫描/解码/Sum早筛/省略MiBYOLO Post/NMSRGA/失败MPP编码 ms/失败/RGA帧/RGA失败/CPU回退MPP解码/失败/回退Copy-out ms/解码/复制/RGA帧/CPU回退/早丢/失败OSD msPublish ms首帧平均/进程最大ms预览流/发布器峰值原始/算法预览峰值SRS流/客户端峰值启动/停止/失败增量

分任务汇总

diff --git a/tools/scenario-bench/src/step-evaluator.js b/tools/scenario-bench/src/step-evaluator.js index d71fb132c..fdd5a609c 100644 --- a/tools/scenario-bench/src/step-evaluator.js +++ b/tools/scenario-bench/src/step-evaluator.js @@ -294,12 +294,25 @@ function summarizeMediaStages(ticks, inferMs) { 'rknnRgaResizeColorMs', 'rknnRgaResizeColorCalls', ), + rknnRgaCropResizeAvgMs: counterAverage( + 'rknnRgaCropResizeMs', + 'rknnRgaCropResizeCalls', + ), + rknnRgaCropResizeCalls: counterDelta('rknnRgaCropResizeCalls'), + rknnRgaCropResizeFailures: counterDelta('rknnRgaCropResizeFailures'), + rknnRgaCropDmaBufFrames: counterDelta('rknnRgaCropDmaBufFrames'), + rknnRgaCropHostFallbacks: counterDelta('rknnRgaCropHostFallbacks'), rknnRgaFailures: counterDelta('rknnRgaFailures'), rknnCpuResizeFallbackAvgMs: counterAverage( 'rknnCpuResizeFallbackMs', 'rknnCpuResizeFallbackCalls', ), rknnCpuResizeFallbacks: counterDelta('rknnCpuResizeFallbackCalls'), + rknnCpuCropResizeFallbackAvgMs: counterAverage( + 'rknnCpuCropResizeFallbackMs', + 'rknnCpuCropResizeFallbackCalls', + ), + rknnCpuCropResizeFallbacks: counterDelta('rknnCpuCropResizeFallbackCalls'), rknnCpuNormalizeFallbackAvgMs: counterAverage( 'rknnCpuNormalizeFallbackMs', 'rknnCpuNormalizeFallbackCalls', @@ -401,6 +414,12 @@ function summarizeMediaStages(ticks, inferMs) { mppCopyOutAvgMs: counterAverage('mppCopyOutMs', 'mppCopyOutFrames'), mppCopyOutFrames: counterDelta('mppCopyOutFrames'), mppCopyOutFailures: counterDelta('mppCopyOutFailures'), + mppRgaCopyOutFrames: counterDelta('mppRgaCopyOutFrames'), + mppRgaCopyOutFailures: counterDelta('mppRgaCopyOutFailures'), + mppCpuCopyOutFallbacks: counterDelta('mppCpuCopyOutFallbacks'), + mppRgaCopyInFrames: counterDelta('mppRgaCopyInFrames'), + mppRgaCopyInFailures: counterDelta('mppRgaCopyInFailures'), + mppCpuCopyInFallbacks: counterDelta('mppCpuCopyInFallbacks'), mppEarlyDroppedFrames: counterDelta('mppEarlyDroppedFrames'), osdAvgMs: counterAverage('osdMs', 'osdFrames'), publishAvgMs: counterAverage('publishMs', 'publishedFrames'), diff --git a/tools/scenario-bench/test/hardware-metrics.test.js b/tools/scenario-bench/test/hardware-metrics.test.js index db8e3d378..0c607fd4f 100644 --- a/tools/scenario-bench/test/hardware-metrics.test.js +++ b/tools/scenario-bench/test/hardware-metrics.test.js @@ -234,9 +234,16 @@ test('step summary derives platform-neutral preview timings and lifecycle deltas rknnRgaFillMs: index * 2, rknnRgaResizeColorCalls: index * 4, rknnRgaResizeColorMs: index * 8, + rknnRgaCropResizeCalls: index * 6, + rknnRgaCropResizeMs: index * 3, + rknnRgaCropResizeFailures: 0, + rknnRgaCropDmaBufFrames: index * 6, + rknnRgaCropHostFallbacks: 0, rknnRgaFailures: 0, rknnCpuResizeFallbackCalls: 0, rknnCpuResizeFallbackMs: 0, + rknnCpuCropResizeFallbackCalls: 0, + rknnCpuCropResizeFallbackMs: 0, rknnCpuNormalizeFallbackCalls: 0, rknnCpuNormalizeFallbackMs: 0, rknnNativeInputMapCalls: index * 4, @@ -304,6 +311,12 @@ test('step summary derives platform-neutral preview timings and lifecycle deltas mppCopyOutFrames: index * 6, mppCopyOutMs: index * 24, mppCopyOutFailures: 0, + mppRgaCopyOutFrames: index * 6, + mppRgaCopyOutFailures: 0, + mppCpuCopyOutFallbacks: 0, + mppRgaCopyInFrames: index * 10, + mppRgaCopyInFailures: 0, + mppCpuCopyInFallbacks: 0, mppEarlyDroppedFrames: index * 4, activePreviewStreams: 1, activePreviewPublishers: 1, diff --git a/tools/scenario-bench/test/longrun-auditor.test.js b/tools/scenario-bench/test/longrun-auditor.test.js index f3fe6769f..c1240db23 100644 --- a/tools/scenario-bench/test/longrun-auditor.test.js +++ b/tools/scenario-bench/test/longrun-auditor.test.js @@ -21,12 +21,18 @@ function accelerator(index) { graphForwardFailures: 0, mppCopyOutFailures: 0, mppCopyOutFrames: frames * 2, + mppRgaCopyOutFailures: 0, + mppRgaCopyOutFrames: frames * 2, + mppCpuCopyOutFallbacks: 0, mppDecodeFailures: 0, mppDecodeFallbacks: 0, mppDecodedFrames: frames * 4, mppEarlyDroppedFrames: frames * 2, mppEncodeFailures: 0, mppEncodedFrames: frames * 3, + mppRgaCopyInFailures: 0, + mppRgaCopyInFrames: frames * 3, + mppCpuCopyInFallbacks: 0, osdFrames: frames * 3, previewStreamFailures: 0, publishedFrames: frames * 3, @@ -35,14 +41,44 @@ function accelerator(index) { rgaFailures: 0, rgaFrames: frames, rknnForwardFailures: 0, + rknnBoundInputBindFailures: 0, + rknnBoundInputCopyCalls: 0, + rknnBoundInputCopyBytes: 0, + rknnBoundInputCopyFailures: 0, + rknnBoundInputSyncCalls: frames * 6, + rknnBoundInputSyncFailures: 0, rknnRgaFailures: 0, + rknnRgaCropResizeCalls: frames, + rknnRgaCropResizeFailures: 0, + rknnRgaCropDmaBufFrames: frames, + rknnRgaCropHostFallbacks: 0, + rknnCpuResizeFallbackCalls: 0, + rknnCpuCropResizeFallbackCalls: 0, + rknnCpuNormalizeFallbackCalls: 0, + rknnDetectorForwardFailures: 0, + rknnDetectorInputsSetCalls: 0, + rknnFloatInputs: 0, + rknnInputCompatibilityFallbacks: 0, + rknnInputsSetCalls: 0, + rknnNativeInputMapCalls: 0, + rknnOutputCompatibilityFallbacks: 0, rknnRgaBoundInputFrames: frames * 3, + rknnRgaBoundInputBindFailures: 0, rknnRgaBoundUint8Frames: frames * 3, + rknnRgaBoundNativeInt8Frames: 0, + rknnRgaBoundRequantizeCalls: 0, + rknnRgaBoundRequantizeMs: 0, rknnRgaBoundInputImportFailures: 0, rknnRgaBoundRequantizeFailures: 0, + rknnMppDmaBufFrames: frames, + rknnMppDmaBufImportCalls: frames * 3, + rknnMppDmaBufImportFailures: 0, + rknnMppDmaBufFallbacks: 0, + rknnUint8ContractInputs: 0, + rknnYolov8DirectCandidateFailures: 0, rknnForwards: frames * 3, - videoDecoderBackend: 'rockchip-copy-out', - videoEncoderBackend: 'rockchip-copy-first', + videoDecoderBackend: 'rockchip-mpp-rga', + videoEncoderBackend: 'rockchip-mpp-rga', }; } @@ -96,10 +132,24 @@ function runResult(count = 5) { previewProfile: { mode: 'algorithm' }, thresholds: { pass: { avgDiscardRate: 0.05, maxDiskUsedPercent: 90 } }, steps: [{ index: 0, channels: 4, holdSec: 259200 }], - samples: Array.from({ length: count }, (_, index) => sample(index + 1)), + samples: [ + sample(0, { phase: 'ramp' }), + ...Array.from({ length: count }, (_, index) => sample(index + 1)), + ], }; } +function enableNativeInt8(input) { + for (const [index, item] of input.samples.entries()) { + const frames = 1000 + (index + 1) * 100; + item.hardware.accelerator.rknnRgaBoundUint8Frames = 0; + item.hardware.accelerator.rknnRgaBoundNativeInt8Frames = frames * 3; + item.hardware.accelerator.rknnRgaBoundRequantizeCalls = frames * 3; + item.hardware.accelerator.rknnRgaBoundRequantizeMs = frames * 3 * 0.8; + } + return input; +} + const options = { gateHours: 4 / 60, nowMs: START + 5 * 60_000 + 10_000, @@ -107,8 +157,8 @@ const options = { maxFreshnessSec: 120, minFpsRatio: 0.9, expectedPreviewStreams: 4, - expectedDecoderBackend: 'rockchip-copy-out', - expectedEncoderBackend: 'rockchip-copy-first', + expectedDecoderBackend: 'rockchip-mpp-rga', + expectedEncoderBackend: 'rockchip-mpp-rga', minRgaBoundUint8Frames: 1, }; @@ -128,6 +178,24 @@ test('long-run audit reports IN_PROGRESS before the wall-clock gate', () => { assert.deepEqual(result.failures, []); }); +test('long-run audit does not count partial-load ramp time toward the duration gate', () => { + const input = runResult(); + input.samples[0].activeChannels = 1; + input.samples[0].channels = input.samples[0].channels.slice(0, 1); + + const running = auditLongRun(input, { ...options, gateHours: 4.5 / 60 }); + assert.equal(running.verdict, 'IN_PROGRESS'); + assert.equal(running.gate.runAgeSec, 300); + assert.equal(running.gate.fullLoadCoverageSec, 240); + assert.equal(running.gate.firstFullLoadAt, new Date(START + 60_000).toISOString()); + + input.status = 'completed'; + input.endedAt = new Date(START + 5 * 60_000).toISOString(); + const completed = auditLongRun(input, { ...options, gateHours: 4.5 / 60 }); + assert.equal(completed.verdict, 'FAIL'); + assert.ok(completed.failures.includes('gate.duration')); +}); + test('completed long-run uses completedAt with a bounded final sampling delay', () => { const input = runResult(); input.status = 'completed'; @@ -165,6 +233,31 @@ test('long-run audit fails native failure increments and unhealthy preview', () assert.ok(result.failures.includes('preview.health')); }); +test('long-run audit rejects a failure counter that is non-zero before sampling begins', () => { + const input = runResult(); + for (const item of input.samples) { + item.hardware.accelerator.mppCpuCopyOutFallbacks = 2; + } + + const result = auditLongRun(input, options); + assert.equal(result.verdict, 'FAIL'); + assert.ok(result.failures.includes('native.failures')); + const nativeFailures = result.checks.find((item) => item.id === 'native.failures'); + assert.equal(nativeFailures.actual.mppCpuCopyOutFallbacks, 2); +}); + +test('long-run audit discovers new failure counters and requires every sample to expose them', () => { + const input = runResult(); + input.samples[1].hardware.accelerator.experimentalHardwareFallbacks = 0; + + const result = auditLongRun(input, options); + assert.equal(result.verdict, 'FAIL'); + assert.ok(result.failures.includes('native.countersPresent')); + assert.ok(result.nativeMedia.dynamicFailureCounterKeys.includes('experimentalHardwareFallbacks')); + const countersPresent = result.checks.find((item) => item.id === 'native.countersPresent'); + assert.ok(countersPresent.actual.includes('experimentalHardwareFallbacks')); +}); + test('memory-pool warm-up keeps cold growth visible and gates the steady-state window', () => { const input = runResult(7); const base = 400 * 1024 * 1024; @@ -213,6 +306,80 @@ test('long-run audit fails when the fused UINT8 bound-input path is inactive', ( assert.ok(result.failures.includes('native.rgaBoundUint8')); }); +test('long-run audit gates complete native INT8 DMA-BUF accounting and transform latency', () => { + const input = enableNativeInt8(runResult()); + const nativeOptions = { + ...options, + minRgaBoundUint8Frames: undefined, + minRgaBoundNativeInt8Frames: 1, + minRknnMppDmaBufFrames: 1, + minRknnRgaCropDmaBufFrames: 1, + maxRgaBoundRequantizeAvgMs: 1, + }; + + const result = auditLongRun(input, nativeOptions); + assert.equal(result.verdict, 'PASS'); + assert.equal(result.checks.find((item) => item.id === 'native.rgaBoundNativeInt8')?.status, 'PASS'); + assert.equal(result.checks.find((item) => item.id === 'native.rknnMppDmaBuf')?.status, 'PASS'); + assert.equal(result.checks.find((item) => item.id === 'native.rknnRgaCropDmaBuf')?.status, 'PASS'); + assert.equal(result.checks.find((item) => item.id === 'native.rgaBoundRequantizeLatency')?.actual, 0.8); + assert.equal(result.checks.find((item) => item.id === 'native.legacyInputPaths')?.status, 'PASS'); +}); + +test('native INT8 audit rejects legacy host-copy and rknn_inputs_set activity', () => { + const input = enableNativeInt8(runResult()); + input.samples[2].hardware.accelerator.rknnBoundInputCopyCalls = 1; + input.samples[2].hardware.accelerator.rknnBoundInputCopyBytes = 1_228_800; + input.samples[2].hardware.accelerator.rknnInputsSetCalls = 1; + + const result = auditLongRun(input, { + ...options, + minRgaBoundUint8Frames: undefined, + minRgaBoundNativeInt8Frames: 1, + }); + assert.equal(result.verdict, 'FAIL'); + assert.ok(result.failures.includes('native.legacyInputPaths')); + const legacyPaths = result.checks.find((item) => item.id === 'native.legacyInputPaths'); + assert.equal(legacyPaths.actual.nonZero.rknnBoundInputCopyCalls, 1); + assert.equal(legacyPaths.actual.nonZero.rknnBoundInputCopyBytes, 1_228_800); + assert.equal(legacyPaths.actual.nonZero.rknnInputsSetCalls, 1); +}); + +test('native INT8 audit permits positive DMA-BUF import and cache-sync counters', () => { + const input = enableNativeInt8(runResult()); + assert.ok(input.samples.at(-1).hardware.accelerator.rknnMppDmaBufImportCalls > 0); + assert.ok(input.samples.at(-1).hardware.accelerator.rknnBoundInputSyncCalls > 0); + + const result = auditLongRun(input, { + ...options, + minRgaBoundUint8Frames: undefined, + minRgaBoundNativeInt8Frames: 1, + }); + assert.equal(result.verdict, 'PASS'); + assert.equal(result.checks.find((item) => item.id === 'native.legacyInputPaths')?.status, 'PASS'); +}); + +test('long-run audit rejects uncovered RKNN forwards and slow native transform', () => { + const input = runResult(); + for (const [index, item] of input.samples.entries()) { + const frames = 1000 + (index + 1) * 100; + item.hardware.accelerator.rknnRgaBoundUint8Frames = 0; + item.hardware.accelerator.rknnRgaBoundNativeInt8Frames = frames * 2; + item.hardware.accelerator.rknnRgaBoundRequantizeCalls = frames * 2; + item.hardware.accelerator.rknnRgaBoundRequantizeMs = frames * 2 * 1.5; + } + + const result = auditLongRun(input, { + ...options, + minRgaBoundUint8Frames: undefined, + minRgaBoundNativeInt8Frames: 1, + maxRgaBoundRequantizeAvgMs: 1, + }); + assert.equal(result.verdict, 'FAIL'); + assert.ok(result.failures.includes('native.rgaBoundNativeInt8')); + assert.ok(result.failures.includes('native.rgaBoundRequantizeLatency')); +}); + test('file audit binds source and allowlisted candidate identity by SHA-256', () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'scenario-longrun-audit-')); try {