data: publish complete Calculet NPU research archive

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt.h

@@ -0,0 +1,302 @@
+#ifndef _CALRT_H_
+#define _CALRT_H_
+
+/**
+ * @brief an interface for c file
+ * 
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stddef.h>
+
+#include "calrt_platform.h"
+#include "calrt_error.h"
+
+#define CAL_MAX_FILE_PATH (128)
+#define CAL_MAX_NAME (256)
+
+// ---------------------------------------- enum --------------------------------------
+
+typedef enum _CalrtDeviceType_e
+{
+    UNDEFINE_TYPE = 0,
+    PCIE = 1,
+    USB,
+    EMU
+}cal_device_type_e;
+
+typedef enum _CalrtTransDirection_e
+{
+    HOST_TO_DEVICE = 0,
+    DEVICE_TO_HOST
+}cal_direction_e;
+
+typedef enum _CalrtPowerMode_e
+{
+    BALANCE = 1,
+    HIGH_PERFORMANCE = 2
+}cal_power_e;
+
+// ---------------------------------------- struct --------------------------------------
+
+typedef struct _CalrtDeviceId_t
+{
+    cal_device_type_e dev_type;
+    char path[CAL_MAX_FILE_PATH];
+}cal_device_id_t;
+
+typedef struct _CalrtDeviceInfo_t
+{
+    cal_device_id_t id;
+    char name[CAL_MAX_NAME];
+    uint64_t ddr_size;
+    float sram_freq_mhz;
+    float sram_size_mb;
+}cal_device_info_t;
+
+typedef struct _CalrtModel_t
+{
+    char name[CAL_MAX_NAME];
+}cal_model_t;
+
+typedef struct _CalrtTensor_t
+{
+    uint64_t size;
+    uint64_t element_num;
+    uint64_t *shape;
+    int shape_dim; // indicate shape length
+}cal_tensor_info_t;
+
+// ---------------------------------------- pre-declare struct --------------------------------------
+
+typedef struct _Calrt_Calbin *cal_calbin;
+typedef struct _Calrt_Dev *cal_device;
+typedef struct _Calrt_InputBuf *cal_ibuffer;
+typedef struct _Calrt_OutputBuf *cal_obuffer;
+typedef struct _Calrt_Tensor *cal_tensor;
+
+
+// ---------------------------------------- Calbin ----------------------------------------
+
+/**
+ * @brief Create a calbin object
+ * 
+ * @param calbin calbin struct
+ * @param calbin_path path to calbin file
+ * @return CalrtError_e 0 on success
+ */
+CALRT_API CalrtError_e create_calbin(cal_calbin *calbin, const char* calbin_path);
+
+/**
+ * @brief Get the all models object
+ * 
+ * @param calbin 
+ * @return int the number of models inside calbin
+ */
+CALRT_API int get_models_number(cal_calbin calbin);
+
+/**
+ * @brief Get the all models name
+ * 
+ * @param model_num the size of models name list
+ * @param name model name list
+ * @return CalrtError_e CalrtSuccess on success(0)
+ */
+CALRT_API CalrtError_e get_all_models(int model_num, char** name, cal_calbin calbin);
+
+CALRT_API void print_calbin(cal_calbin calbin);
+
+// ---------------------------------------- Device ----------------------------------------
+
+/**
+ * @brief Create a pcie device object
+ * 
+ * @param device 
+ * @return CalrtSuccess on success(0)
+ */
+CALRT_API CalrtError_e create_device(cal_device *device);
+
+/**
+ * @brief Create a device by type object
+ * 
+ * @param device cal_device
+ * @param type device type
+ * @return CalrtError_e 
+ */
+CALRT_API CalrtError_e create_device_by_type(cal_device *device, cal_device_type_e type);
+
+CALRT_API void reset_device_configuration(cal_device device);
+
+CALRT_API void reset_device(cal_device device);
+
+// ---------------------------------------- Buffer ----------------------------------------
+
+/**
+ * @brief Create a input buffer object
+ * 
+ * @param buffer 
+ * @param calbin
+ * @return CalrtError_e 0 on success
+ */
+CALRT_API CalrtError_e create_input_buffer(cal_ibuffer *buffer, cal_calbin calbin, const char* model_name);
+
+/**
+ * @brief Create a output buffer object
+ * 
+ * @param buffer 
+ * @param calbin
+ * @return CalrtError_e CalrtSuccess on success(0)
+ */
+CALRT_API CalrtError_e create_output_buffer(cal_obuffer *buffer, cal_calbin calbin, const char* model_name);
+
+/**
+ * @brief Get the input tensor number
+ * 
+ * @param buffer input buffer
+ * @return the amount of tensors
+ */
+CALRT_API int get_input_tensor_num(cal_ibuffer buffer);
+
+/**
+ * @brief Get the output tensor number
+ * 
+ * @param buffer output buffer
+ * @return the amount of tensors
+ */
+CALRT_API int get_output_tensor_num(cal_obuffer buffer);
+
+/**
+ * @brief Get the input tensor by name
+ * 
+ * @param buffer input buffer struct
+ * @param name tensor name
+ * @param tensor cal_tensor
+ * @return CalrtSuccess on success(0) 
+ */
+CALRT_API CalrtError_e get_input_tensor_by_name(cal_ibuffer buffer, const char* name, cal_tensor *tensor);
+
+/**
+ * @brief Get the output tensor by name
+ * 
+ * @param buffer output buffer struct
+ * @param name tensor name
+ * @param tensor 
+ * @return CalrtSuccess on success(0)
+ */
+CALRT_API CalrtError_e get_output_tensor_by_name(cal_obuffer buffer, const char* name, cal_tensor *tensor);
+
+/**
+ * @brief Set the csr value by name
+ * 
+ * @param buffer 
+ * @param name 
+ * @param value 
+ */
+CALRT_API void set_csr_by_name(cal_ibuffer buffer, const char* name, uint32_t value);
+
+/**
+ * @brief Get the csr value by name
+ * 
+ * @param buffer 
+ * @param name 
+ * @return value 
+ */
+CALRT_API uint32_t get_csr_value_by_name(cal_ibuffer buffer, const char* name);
+
+CALRT_API cal_tensor_info_t get_tensor_info(cal_tensor tensor);
+
+// ---------------------------------------- Inference ----------------------------------------
+
+/**
+ * @brief blocking inference interface
+ * 
+ * @param device 
+ * @param model_name 
+ * @param calbin 
+ * @param inputBuffer 
+ * @param outputBuffer 
+ */
+CALRT_API void block_infer_model(cal_device device, const char* model_name, cal_calbin calbin, cal_ibuffer inputBuffer, cal_obuffer outputBuffer);
+
+/**
+ * @brief non-blocking inference interface
+ * 
+ * @param device 
+ * @param model_name 
+ * @param calbin 
+ * @param inputBuffer 
+ * @param outputBuffer 
+ */
+CALRT_API void non_block_infer_model(cal_device device, const char* model_name, cal_calbin calbin, cal_ibuffer inputBuffer, cal_obuffer outputBuffer);
+
+/**
+ * @brief co-op with non_block_infer_model api
+ * 
+ * @param outputBuffer 
+ * @return CalrtSuccess on success(0)
+ */
+CALRT_API CalrtError_e wait_infer_done(cal_obuffer outputBuffer);
+
+// ---------------------------------------- Configure ----------------------------------------
+
+/**
+ * @brief configure calbin into the current device
+ * 
+ * @param device 
+ * @param calbin 
+ * @return CalrtSuccess on success(0)
+ */
+CALRT_API CalrtError_e configure_device(cal_device device, cal_calbin calbin);
+
+// ---------------------------------------- Memory Management ----------------------------------------
+
+/**
+ * @brief copy host data to device. tensor MUST be input tensor
+ * @warning user must guarantee tensor is from input buffer
+ * 
+ * @param host host side buffer
+ * @param tensor
+ * @param size byte
+ */
+CALRT_API CalrtError_e cal_copy_mem(void *host, cal_tensor tensor, uint64_t size, cal_direction_e direction);
+
+/**
+ * @brief safer way to copy host data to device.
+ * 
+ * @param host host side buffer
+ * @param size byte
+ * @param buffer input buffer
+ * @param name tensor name
+ * @param offset the location where to write data in device memory
+ * @return CalrtSuccess on success(0)
+ */
+CALRT_API CalrtError_e copy_mem_to_device_by_tensor_name(void *host, cal_ibuffer ibuffer, uint64_t size, const char* name, uint64_t offset=0);
+
+/**
+ * @brief safer way to copy device data to host. tensor must be output tensor
+ * 
+ * @param host host side buffer
+ * @param size byte
+ * @param buffer output buffer
+ * @param name tensor name
+ * @param offset the location where to read data in device memory
+ */
+CALRT_API CalrtError_e copy_mem_to_host_by_tensor_name(void *host, cal_obuffer obuffer, uint64_t size, const char* name, uint64_t offset=0);
+
+// ---------------------------------------- Resource Release ----------------------------------------
+
+CALRT_API void release_device(cal_device device);
+CALRT_API void release_calbin(cal_calbin calbin);
+CALRT_API void release_input_buffer(cal_ibuffer buffer);
+CALRT_API void release_output_buffer(cal_obuffer buffer);
+CALRT_API void release_tensor_info(cal_tensor_info_t &tensor_info);
+
+#ifdef __cplusplus  
+}
+#endif
+
+#endif /*_CALRT_H_*/

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt.hpp

@@ -0,0 +1,22 @@
+/**
+ * @file calrt.hpp
+ * @author your name (you@domain.com)
+ * @brief Calculet runtime C++ API
+ * @version 0.1
+ * @date 2025-08-13
+ * 
+ * @copyright Copyright (c) 2025
+ * 
+ */
+
+#pragma once
+
+#include "calrt_buffer.h"
+#include "calrt_calbin.h"
+#include "calrt_deploy.h"
+#include "calrt_device.h"
+#include "calrt_error.h"
+#include "calrt_infer.h"
+#include "calrt_utils.h"
+#include "calrt_vdevice.h"
+#include "protocol.hpp"

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_buffer.h

@@ -0,0 +1,170 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <memory>
+#include <unordered_map>
+#include <atomic>
+#include <condition_variable>
+#include <mutex>
+
+#include "calrt_utils.h"
+
+namespace calrt
+{
+    class CalrtJob;
+
+    struct TensorPkg
+    {
+        CalrtTensor *tensor = nullptr;
+        uint64_t address[2]{0};
+
+        // TensorPkg() = default;
+        // TensorPkg(const TensorPkg&) = delete;
+        // TensorPkg& operator=(const TensorPkg&) = delete;
+        // TensorPkg(TensorPkg&&) noexcept = default;
+        // TensorPkg& operator=(TensorPkg&&) noexcept = default;
+        ~TensorPkg();
+    };
+
+    struct CALRT_API ModelHyperParameters_s
+    {
+        /**
+         * @brief Get the Csr Value By Name object
+         * 
+         * @param csrName 
+         * @return uint32_t 0: for error msg
+         */
+        uint32_t GetCsrValueByName(const std::string &csrName);
+        CalrtError_e SetCsrByName(const std::string &csrName, uint32_t value);
+
+        private:
+            friend class CalrtInputBuf;
+            void init(const std::string &csrName, uint32_t value);
+            std::unordered_map<std::string, uint32_t> csr; // csr | value
+    };
+
+    class CALRT_API CalrtInputBuf
+    {
+    public:
+        static std::unique_ptr<CalrtInputBuf> CreateInputBuf(CalrtBufferInfo_s &&info);
+        CalrtInputBuf(CalrtBufferInfo_s &&info);
+        CalrtInputBuf(const CalrtInputBuf &) = delete;
+        CalrtInputBuf& operator=(const CalrtInputBuf&) = delete;
+        CalrtInputBuf(CalrtInputBuf &&) noexcept = default;
+        CalrtInputBuf &operator=(CalrtInputBuf &&other) noexcept = default;
+        
+        ~CalrtInputBuf();
+
+        std::unordered_map<std::string, CalrtTensor*> GetTensors();
+        CalrtTensor* GetTensorByName(const std::string &tensorName);
+
+        /**
+         * @brief slice tensor by offset and size for the specific data transfer
+         * 
+         * @param src host side buffer
+         * @param tensorName 
+         * @param offset 
+         * @param size 
+         * @return CalrtSuccess on success  
+         */
+        CalrtError_e SliceTensorByName(void *src, const std::string &tensorName, uint64_t offset, uint64_t size);
+        CalrtError_e ResetTensorByName(const std::string &tensorName);
+        void ResetAllTensors();
+        std::vector<CalrtDevBuf_s>& GetTensorMemInfo() { return mDevBufs; }
+
+        /**
+         * @brief Get the current modifiable HyperParam struct
+         * 
+         * @return ModelHyperParameters_s& 
+         */
+        ModelHyperParameters_s &GetCurHyperParam() {return mHyperParam;}
+        
+        std::string Name();
+
+        size_t GetTensorNum();
+
+        float GetInputTransferTime();
+
+    private:
+        friend class CalrtJob;
+
+        CalrtBufferInfo_s mInfo;
+        std::vector<CalrtDevBuf_s> mDevBufs;
+        ModelHyperParameters_s mHyperParam;
+        std::vector<TensorPkg> mTensorPkgs;
+        std::unordered_map<std::string, uint32_t> mTensorNameMap;
+        float mTransTime;
+
+        std::vector<TensorPkg> &GetTensorPkg() {return mTensorPkgs;}
+        const std::vector< std::pair<std::string, uint32_t> > &GetCsrTable() const {return mInfo.csrTable;}
+        void SetTransTime(float transTime) noexcept;
+    };
+
+    class CALRT_API CalrtOutputBuf
+    {
+    public:
+        static std::unique_ptr<CalrtOutputBuf> CreateOutputBuf(CalrtBufferInfo_s &&info);
+        CalrtOutputBuf(CalrtBufferInfo_s &&info);
+        CalrtOutputBuf(const CalrtOutputBuf &) = delete;
+        CalrtOutputBuf& operator=(const CalrtOutputBuf&) = delete;
+        CalrtOutputBuf(CalrtOutputBuf &&) = delete;
+        CalrtOutputBuf &operator=(CalrtOutputBuf &&other) = delete;
+
+        enum class CalrtOutputBufStatus_e : int32_t {
+            CALRT_OBUF_STATUS_DEFAULT = 0,
+            CALRT_OBUF_STATUS_PENDING,  // in sw start queue, not started yet
+            CALRT_OBUF_STATUS_RUNNING,  // in calcore, running
+            CALRT_OBUF_STATUS_DONE,      // in sw done queue, done
+            CALRT_OBUF_STATUS_DONE_CCU_EXCEPTION
+        };
+
+        ~CalrtOutputBuf();
+
+        void SetStatus(CalrtOutputBufStatus_e status);
+        CalrtOutputBufStatus_e GetStatus();
+
+        std::unordered_map<std::string, CalrtTensor*> GetTensors();
+        CalrtTensor* GetTensorByName(const std::string &tensorName);
+
+        /**
+         * @brief slice tensor by offset and size for the specific data transfer
+         * 
+         * @param src host side buffer
+         * @param tensorName 
+         * @param offset 
+         * @param size 
+         * @return CalrtSuccess on success 
+         */
+        CalrtError_e SliceTensorByName(void *src, const std::string &tensorName, uint64_t offset, uint64_t size);
+        CalrtError_e ResetTensorByName(const std::string &tensorName);
+        void ResetAllTensors();
+
+        void Notify(CalrtOutputBuf::CalrtOutputBufStatus_e status);
+        CalrtError_e Wait();
+
+        CalrtBufferInfo_s GetBufferInfo();
+        std::string Name();
+        void Reset();
+        size_t GetTensorNum();
+        float GetWaitTime();
+        float GetOutputTransferTime();
+
+    private:
+        friend class CalrtJob;
+
+        CalrtBufferInfo_s mInfo;
+        std::atomic<int32_t> mStatus {0};
+        std::vector<TensorPkg> mTensorPkgs;
+        std::unordered_map<std::string, uint32_t> mTensorNameMap;
+        float mWaitTime;
+        float mTransTime;
+        std::condition_variable mDone;
+        std::mutex mLock;
+
+        std::vector<TensorPkg> &GetTensorPkg();
+        void SetWaitTime(float time) noexcept;
+        void SetTransTime(float transTime) noexcept;
+    };
+} // namespace calrt

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_calbin.h

@@ -0,0 +1,93 @@
+#pragma once
+
+#include <memory>
+#include <unordered_map>
+#include <map>
+#include <vector>
+#include <filesystem>
+#include <optional>
+#include <cstdint>
+#include <string>
+#include <string_view>
+
+#include "calrt_utils.h"
+#include "elf_parser.hpp"
+
+namespace calrt
+{
+    class CALRT_API Calbin final
+    {
+        public:
+            /**
+             * @brief Create a Parser object
+             * 
+             * @param path 
+             * @return std::unique_ptr<Calbin> 
+             */
+            static std::unique_ptr<Calbin> CreateCalbin(const std::string &path);
+            Calbin(const Calbin&) = delete;
+            Calbin& operator=(const Calbin&) = delete;
+            Calbin(Calbin &&) = default;
+            Calbin &operator=(Calbin &&other) = default;
+            ~Calbin() = default;
+
+            CalrtCalbin& GetCalbinBrief();
+
+            CalbinModel* GetModelByName(std::string_view modelName);
+            std::vector<CalbinModel> *GetAllModels();
+
+            /**
+             * @brief Get the Model By Type object
+             * @note type: prefill, decode, kv_update
+             * 
+             * @param type 
+             * @return std::vector<CalbinModel*>
+             */
+            std::vector<CalbinModel*> GetModelByType(std::string_view type);
+
+            std::vector<calc_efl::Parsed_Elf_s> &GetPairedElf(const std::string &model, const std::vector<uint32_t> &chipMask);
+
+            CalbinModel& GetGlobalMemInfo();
+            const CalbinLLM_s& GetLLMInfo() const;
+            
+            /**
+             * @brief print out calbin essential info, llm info and kv-cache info.
+             * 
+             */
+            void Report();
+
+            CalbinSectionPlace_e GetStackLoc(const std::string &modelName);
+
+            const std::string &Version() const { return mVersion; }
+
+            uint64_t GetModelWorloadByName(const std::string &modelName);
+
+            std::vector<uint8_t> GetGoldenInputByModelName(const std::string &modelName);
+            std::vector<uint8_t> GetGoldenOutputByModelName(const std::string &modelName);
+            std::string GetRootPath() {return mRootPath;}
+            // std::pair<uint64_t, std::vector<uint8_t>> GetGoldenInputWithAddr(const char* modelName);
+            // std::pair<uint64_t, std::vector<uint8_t>> GetGoldenOutputWithAddr(const char* modelName);
+
+        private:
+            Calbin(std::string &&rootPath);
+            CalrtError_e GenCalrtCalbin(const std::unordered_map<std::string, std::vector<std::filesystem::path>> &modelMap);
+            void SetMaxElfSize(const std::string &modelName);
+            CalrtError_e InitSection(const std::vector<std::string> &vals, CalbinModel &mod, const std::string &fullTag, const std::string &filePath);
+            CalrtError_e RecordGlobalMem(const std::string &path);
+
+            CalrtError_e ParseMemFile(CalbinModel &tempModel, CalbinLLM_s &LLMInfo_, const std::filesystem::path &filePath);
+            CalrtError_e Validation();
+
+            std::unordered_map<std::string, CalbinSectionPlace_e> mStackLoc;
+            std::unordered_map<std::string, std::unordered_map<uint64_t, std::vector<calc_efl::Parsed_Elf_s> >> mModelPairedElf;
+            std::unordered_map<std::string, uint64_t> mModelWorkLoads; // modelName , workload
+
+            std::string mRootPath;
+            CalrtCalbin mCalbin;
+            CalbinModel mGlobalModel;
+            CalbinLLM_s mLLMInfo;
+            int32_t mCompatibility; // indicate compitable device model type: ks01 = 0, ks02 = 1.
+            std::string mVersion;
+            uint64_t mCalbinSize; // require for device memory size
+    };
+} // namespace calrt

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_deploy.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <cassert>
+
+#include "calrt_utils.h"
+#include "calrt_vdevice.h"
+#include "calrt_calbin.h"
+
+namespace calrt
+{
+
+// template<typename CalculetDevice>
+// CalrtError_e configure(CalculetDevice *vDev, Calbin *pParser)
+// {
+//     using DECAYTYPE = std::decay_t<CalculetDevice>;
+//     REPORT_CALRT_ERROR_IF(CalrtErrorAssert, vDev != nullptr, "Device pointer is nullptr");
+//     if constexpr(std::is_same_v<VirtualDevice, DECAYTYPE>)
+//     {
+//         auto physicalDev = vDev->GetDevice();
+//         return configure_impl(physicalDev.get(), pParser);
+//     }
+//     else if constexpr(std::is_base_of_v<CalrtDevice, DECAYTYPE>)
+//     {
+//         return configure_impl(vDev, pParser);
+//     }
+//     else
+//     {
+//         return CalrtErrorInvalidConfiguration;
+//     }
+// }
+
+    /**
+     * @brief 
+     * 
+     * @param vDevPtr 
+     * @param pCalbin 
+     * @param dumpIni enable dump configuration ini. default: disable
+     * @return CalrtError_e 
+     */
+    CALRT_API CalrtError_e configure(VirtualDevice* vDevPtr, Calbin *pCalbin, bool dumpIni=false);
+
+    /**
+     * @brief 
+     * 
+     * @param devPtr 
+     * @param pCalbin 
+     * @param dumpIni enable dump configuration ini. default: disable
+     * @return CalrtError_e 
+     */
+    CALRT_API CalrtError_e configure(std::unique_ptr<VirtualDevice>& vDevPtr, Calbin *pCalbin, bool dumpIni=false);
+
+    CALRT_API CalbinModel* findModel(std::vector<CalbinModel> &modelList, const char *name);
+}

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_device.h

@@ -0,0 +1,364 @@
+/**
+ * @file calrt_device.h
+ * @author your name (you@domain.com)
+ * @brief 
+ * @version 0.1
+ * @date 2024-11-11
+ * 
+ * @copyright Copyright (c) 2024
+ * 
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <optional>
+#include <vector>
+#include <functional>
+#include <atomic>
+#include <type_traits>
+
+#include "calrt_utils.h"
+#include "calrt_calbin.h"
+
+namespace calrt 
+{
+    enum CalrtDeviceType_e{
+        NO_TYPE = 0,
+        PCIE = 1,
+        USB,
+        EMU
+    };
+
+    enum class DeviceWorkingStatus : int
+    {
+        // 0: busy, 1: AVAILABLE, 2: taken
+        BUSY = 0,
+        AVAILABLE,
+        TAKEN
+    };
+
+    struct CalrtDeviceId_s{
+        // bool emuId = false; //true if users only want emulator which would ignore physical device. Users need to decided if they need to turn on this option.
+        CalrtDeviceType_e devType;
+        std::string devPath;
+        int serialNum = 0; // 0 = ks1, 1 = ks2 ...
+
+        int32_t compare(const CalrtDeviceId_s &rh) {
+            int32_t result = 0;
+            result += (devType == rh.devType) ? 0 : 1;
+            result += (devPath == rh.devPath) ? 0 : 1;
+            return result;
+        }
+    };
+
+    struct CalrtDeviceInfo_s {
+        CalrtDeviceId_s devId;
+        std::string name;
+        uint64_t ddrSize = (32UL << 30);
+        float sramFreqMhz = 1000.0;
+        uint64_t sramSizeMB = 18;
+    };
+
+    struct RemoteProtocol {
+        uint32_t masks[3];
+        char password[128];
+    };
+
+    class CALRT_API CalrtDeviceInterface {
+
+    public:
+        inline static std::function<bool(char *, const uint64_t, const uint64_t, const int32_t)> DeviceMemAccessFunc;
+        static bool DeviceMemAccess(char * calrtData, const uint64_t devAddr, const uint64_t size, const int32_t write_en);
+    };
+
+    class CALRT_API  CalrtDevice
+    {
+
+    friend class VirtualDevice;
+
+    public:
+        /**
+         * @brief Create a Device object. Current support [PCIe], [USB]
+         *
+         * @note always return valid pointer, but self-check if device is valid. Priority PCIe device.
+         * @return std::unique_ptr<CalrtDevice>
+         */
+        static std::unique_ptr<CalrtDevice> CreateDevice();
+
+        /**
+         * @brief Create a PCIe Device
+         *
+         * @note always return valid pointer, but self-check if device is valid.
+         * @param id
+         * @return std::unique_ptr<CalrtDevice>
+         */
+        static std::unique_ptr<CalrtDevice> CreatePCIeDevice();
+
+        /**
+         * @brief Create a Emulator Device
+         * 
+         * @return std::unique_ptr<CalrtDevice> 
+         */
+        static std::unique_ptr<CalrtDevice> CreateEmuDevice();
+
+        /**
+         * @brief used for refresh device manager for any new hotplug devices installation
+         * 
+         */
+        static void RefreshDevice();
+
+        virtual ~CalrtDevice() = default;
+        CalrtDevice(const CalrtDevice &) = delete;
+        CalrtDevice &operator=(const CalrtDevice &) = delete;
+        CalrtDevice(CalrtDevice &&) = delete;
+        CalrtDevice &operator=(CalrtDevice &&other) = delete;
+
+        typedef bool (*DeviceMemAccessPtr)(char *, const uint64_t, const uint64_t, const int32_t);
+
+        virtual void SetDevMemAccessApi(DeviceMemAccessPtr fDeviceMemAccess);
+
+        virtual CalrtError_e RegisterDram(int64_t startAddr, int64_t size, const char *tag="rsvd_dram") = 0;
+
+        virtual CalrtError_e RegisterSyncUnit(int64_t startAddr, int64_t size, const char *tag="rsvd_dram") = 0;
+
+        virtual CalrtError_e RegisterSramBuf(int64_t startAddr, int64_t size, const char *tag="rsvd_dram") = 0;
+
+        CalrtError_e Configure(CalbinModel *pModel, Calbin *pCalbin);
+
+        CalrtError_e Configure(CalbinModel &model, Calbin *pCalbin);
+
+        /**
+         * @brief dump all configure meta data at once
+         * 
+         * @param modelName 
+         * @param pCalbin 
+         * @param tag 
+         */
+        void DumpDeviceConfigureMetaData(const std::string &modelName, Calbin *pCalbin, const std::string &tag);
+
+        /**
+         * @brief allow to dump multiple section at once in "ini" file
+         * 
+         * @tparam CalbinSectionType_e
+         */
+        template <typename... SecType>
+        inline void DumpDeviceConfigToIniFileBySections(const std::string &outputPath, SecType... secType)
+        {
+            static_assert((std::is_same_v<SecType, CalbinSectionType_e> && ...), "Must pass CalbinSectionType_e arguments");
+            std::unordered_map<std::string, std::string> iniFile;
+
+            (RecordDeviceConfig(iniFile, secType), ...);
+
+            DumpMapMirror(outputPath, iniFile, 2);
+        }
+
+        /**
+         * @brief copy host memory to device memory
+         *
+         * @param srcAddr host mem address
+         * @param devAddr device mem address
+         * @param size bytes
+         */
+        virtual CalrtError_e WriteToDevice(void *srcAddr, const uint64_t devAddr, const uint64_t size) = 0;
+
+        /**
+         * @brief copy device memory to host memory
+         *
+         * @param srcAddr host mem address
+         * @param devAddr device mem address
+         * @param size bytes
+         */
+        virtual CalrtError_e ReadFromDevice(void *srcAddr, const uint64_t devAddr, const uint64_t size) = 0;
+
+        virtual CalrtError_e WriteReg(const uint64_t regAddr, uint32_t data) = 0; // only support bar2
+
+        virtual CalrtError_e ReadReg(const uint64_t regAddr, uint32_t &data) = 0; // only support bar2
+
+        // only support bar4
+        virtual CalrtError_e WriteReg64(const uint64_t regAddr, uint64_t data)
+        {
+            return CalrtSuccess;
+        }
+
+        virtual CalrtError_e ReadReg64(const uint64_t regAddr, uint64_t &data)
+        {
+            return CalrtSuccess;
+        }
+
+        virtual uint64_t CreateBuf(uint64_t size, uint32_t alignLog2Byte=0) = 0;
+
+        virtual uint64_t CreateSramBuf(uint64_t size, uint32_t alignLog2Byte=0) = 0;
+
+        virtual uint64_t ApplySyncUnit() = 0;
+
+        virtual void FreeBuf(uint64_t addr) = 0;
+
+        virtual void FreeSramBuf(uint64_t addr) = 0;
+
+        virtual void FreeSyncUnitByAddress(uint64_t addr) = 0;
+
+        virtual void FreeSyncUnitByIndex(uint32_t idx) = 0;
+
+        virtual void ClearAllDevMem() = 0;
+
+        virtual void ClearDynamicMem() = 0;
+
+        //virtual void WaitJobDone(std::shared_ptr<CalrtJob> pJob);
+
+        virtual CalrtError_e Reset() = 0;
+
+        void ResetCCU();
+
+        //TODO it should return a list or struct. TBD
+        virtual CalrtError_e GetFirmwareInfo();
+
+        virtual std::string GetDevInfo() = 0;
+
+        virtual std::unordered_map<int64_t, int64_t> GetReservedMem() = 0;
+
+        virtual void SetPowerMode(PowerMode power) = 0;
+
+        /**
+         * @brief Get the Memory Usage object
+         * 
+         * @return std::pair<float,float> {usedDram, DramTotal} size in MiB
+         */
+        virtual std::pair<float,float> GetMemoryUsage() = 0;
+
+        void SetConfigModel(Calbin* pCalbin);
+
+        const uint64_t GetCurrentCalbinOnDevice();
+
+        int32_t GetNumPendingJob(); // CLI impl func
+
+        int32_t GetNumFinishedJob(); // CLI impl func
+
+        /**
+         * @brief indicating if device is created successfully
+         * 
+         * @return const CalrtError_e 
+         */
+        virtual const CalrtError_e Status() const = 0;
+
+        /**
+         * @brief Set the Status object. indicating if device is created successfully
+         * 
+         */
+        virtual void SetStatus(CalrtError_e status) = 0;
+
+        /**
+         * @brief Get the Device Running Status.
+         * 
+         * @return int 
+         */
+        virtual DeviceWorkingStatus GetDeviceRunningStatus() = 0;
+
+        const std::string &Name() {return mInfo.name;}
+
+        void EnableTraceDevice(bool enable);
+
+        virtual uint64_t GetDeviceLeftJobs() = 0;
+
+        int GetDeviceSerialNum() { return mInfo.devId.serialNum; }
+
+        virtual void Release() = 0;
+
+        uint32_t ReadRemoteReg(uint64_t regAddr, uint32_t chipId);
+
+        CalrtError_e ReadRemoteChipMem(void *srcAddr, const uint64_t devAddr, const uint64_t size, uint32_t chipId);
+
+        CalrtError_e WriteRemoteChipMem(void *srcAddr, const uint64_t devAddr, const uint64_t size, uint32_t chipId);
+
+        void WriteRemoteReg(uint64_t regAddr, uint32_t regValue, uint32_t chipId);
+
+        CalrtDeviceType_e Type() noexcept {return mInfo.devId.devType;}
+
+        void ClearDevLib();
+
+        virtual uint32_t GetExpectedEndQueueEntries() {return 0;}
+
+#ifndef RELRT
+        std::string GetCalbinPath() noexcept;
+#endif
+
+    protected:
+        CalrtDevice(const CalrtDeviceInfo_s &info);
+
+        void StartJob(uint64_t jobAddr, uint64_t jobSize);
+
+        /**
+         * @brief wait job done
+         * 
+         * @return uint32_t ccu work status
+         */
+        uint32_t WaitJobDone(bool singleEntry = false);
+
+        /**
+         * @brief wait job done
+         * 
+         * @param jobEndInfo record [Hi32: infer time consume, Lo32: finished job id]
+         * @return uint32_t ccu work status
+         */
+        uint32_t WaitJobDone(uint64_t& jobEndInfo, bool singleEntry = false);
+        virtual void PollingJobDonePowerUp() = 0;
+        /**
+         * @brief 
+         * 
+         * @param status 0: busy, 1: ready, 2: taken 3: error
+         */
+        virtual void UpdateDeviceRunningStatus(uint32_t status) {};
+        virtual void UpdateDeviceJobs(bool add) {};
+        void CleanJobQ();
+        CalbinSectionPlace_e GetStackPlace(const std::string &modelName);
+
+        CalrtError_e DeployGlobalMem(Calbin *pCalbin);
+        CalrtError_e DeployCPUSection (std::vector<CalbinSection_s> &sectionList, std::vector<int32_t> &tempChips, Calbin *pCalbin);
+        CalrtError_e DeployCCUSection (std::vector<CalbinSection_s> &sectionList, std::vector<int32_t> &tempChips, CalbinModel &model, Calbin *pCalbin);
+        CalrtError_e DeployRodataSection (std::vector<CalbinSection_s> &sectionList, Calbin *pCalbin);
+        CalrtError_e DeployLibSection (std::vector<CalbinSection_s> &sectionList, std::vector<int32_t> &tempChips, CalbinModel &model, Calbin *pCalbin);
+
+        virtual std::vector<std::unordered_map<int64_t, int64_t>> GetDeviceMemoryLayout() = 0;
+        virtual void UpdateBandWidth(float bandwidth) = 0;
+
+        void RecordDeviceConfig(std::unordered_map<std::string, std::string> &iniFile, const CalbinSectionType_e sectionType);
+
+        void DumpMapMirror(const std::string &path, const std::unordered_map<std::string, std::string> &iniFile, int flag);
+
+        /**
+         * @brief 
+         * 
+         * @param num 
+         * @param flag 0: minus, 1: add 2: set 3: reset
+         */
+        virtual void UpdateDevRuntimeInfo (uint32_t num, int flag) = 0;
+
+    private:
+        CalrtDeviceInfo_s mInfo;
+        Calbin *pmCurrentCalbin = nullptr;
+        uint64_t mCurCalbinHash = 0;
+        std::unordered_map<std::string, CalbinSectionPlace_e> mStackPlace;
+        std::unordered_map<std::string, uint64_t> mWorkloadsPerModel;
+        std::atomic<int32_t> mDoneJobs = 0;
+
+        /**
+         * @brief deploy lib for chip 0 and chipx if chip mask contains both chip0 and chipx
+         * 
+         * @param section 
+         * @param modelName 
+         */
+        void DeployLibToX(CalbinSection_s &section, const std::string &modelName);
+
+        /**
+         * @brief deploy lib for chip0 only
+         * 
+         * @param section 
+         * @param modelName 
+         */
+        void DeployLibToHead(CalbinSection_s &section, const std::string &modelName);
+
+        // class DeviceInternal;
+        // std::unique_ptr<DeviceInternal> pmInternal;
+    };
+}

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_error.h

@@ -0,0 +1,13 @@
+#ifndef _CALRT_ERR_H_
+#define _CALRT_ERR_H_
+
+typedef enum CalError_e
+{
+    #define CALRT_ERROR(name, value) name = value,
+        #include "calrt_errorlist.def"
+    #undef CALRT_ERROR
+
+    CalrtStatusAmount
+} CalrtError_e;
+
+#endif // _CALRT_ERR_H_

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_errorlist.def

@@ -0,0 +1,30 @@
+CALRT_ERROR(CalrtSuccess, 0)
+CALRT_ERROR(CalrtErrorMemoryAllocation, 1)
+CALRT_ERROR(CalrtErrorMemoryAlreadyRegistered, 2)
+CALRT_ERROR(CalrtErrorAssert, 3)
+CALRT_ERROR(CalrtErrorInvalidValue, 4)
+CALRT_ERROR(CalrtErrorInvalidConfiguration, 5)
+CALRT_ERROR(CalrtErrorNoDevice, 6)
+CALRT_ERROR(CalrtErrorAlreadyConfigured, 7)
+CALRT_ERROR(CalrtErrorIncompatibleDriver, 8) // update device driver. too old to be supported by runtime
+CALRT_ERROR(CalrtErrorRequireNewerRT, 9) // update runtime library. too old to be compatible with current device
+CALRT_ERROR(CalrtErrorFileNotFound, 10)
+CALRT_ERROR(CalrtErrorBrokenFile, 11)
+CALRT_ERROR(CalrtErrorInvalidELF, 12) // ELF version is invalid
+CALRT_ERROR(CalrtErrorTimeout, 13)
+CALRT_ERROR(CalrtErrorOutOfRange, 14)
+CALRT_ERROR(CalrtErrorNotExpected, 15)
+CALRT_ERROR(CalrtErrorMissConfiguration, 16)
+CALRT_ERROR(CalrtErrorDeviceBusy, 17)
+CALRT_ERROR(CalrtErrorDeviceUnavailable, 18) // device disconnected, PCIe loose contact, power supply low
+CALRT_ERROR(CalrtErrorInvalidCalbin, 19)
+CALRT_ERROR(CalrtErrorUnknown, 20)
+CALRT_ERROR(CalrtErrorInvalidDataType, 21)
+CALRT_ERROR(CalrtErrorInvalidDataShape, 22)
+CALRT_ERROR(CalrtErrorInvalidLibDep, 23)
+CALRT_ERROR(CalrtErrorDeviceMapBroken, 24)
+CALRT_ERROR(CalrtErrorFailedSoftReset, 25)
+CALRT_ERROR(CalrtErrorFailedConnectServer, 26)
+CALRT_ERROR(CalrtErrorDeviceUnsupport, 27) // the current device doesn't support this action
+CALRT_ERROR(CalrtErrorDeviceCrash, 28) // ccu execution error
+CALRT_ERROR(CalrtErrorWrongDirection, 29)

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_infer.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "calrt_utils.h"
+#include "calrt_buffer.h"
+#include "calrt_device.h"
+#include "calrt_vdevice.h"
+
+namespace calrt
+{
+    CALRT_API std::unique_ptr<CalrtInputBuf> createInputBuf(const CalbinModel &model);
+
+    CALRT_API std::unique_ptr<CalrtOutputBuf> createOutputBuf(const CalbinModel &model);
+
+    CALRT_API CalrtError_e infer(std::unique_ptr<VirtualDevice> &vDev, CalbinModel *model, CalrtInputBuf* inputBuffer, CalrtOutputBuf* outputBuffer);
+    CALRT_API CalrtError_e infer(VirtualDevice *vDev, CalbinModel *model, CalrtInputBuf* inputBuffer, CalrtOutputBuf* outputBuffer);
+    CALRT_API CalrtError_e infer_with_fixed_task_type(std::unique_ptr<VirtualDevice> &vDev, CalbinModel *model, CalrtInputBuf* inputBuffer, CalrtOutputBuf* outputBuffer, TaskType_e type);
+    CALRT_API CalrtError_e infer_with_fixed_task_type(VirtualDevice *vDev, CalbinModel *model, CalrtInputBuf* inputBuffer, CalrtOutputBuf* outputBuffer, TaskType_e type);
+}

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_kv.h

@@ -0,0 +1,178 @@
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "calrt_calbin.h"
+#include "calrt_utils.h"
+#include "calrt_vdevice.h"
+
+namespace calrt {
+
+typedef int32_t KvPos;
+typedef int32_t KvSeqId;
+
+struct KvCellAddr_s {
+    int64_t src_offset; // offset from base_addr
+    int64_t dst_offset; // offset from base_addr
+};
+
+enum class KvBatchMode_e : int32_t {
+    ONLY_VALID = 0,
+    ALL = 1, // 0:only mv valid batch ; 1:mv 16 batch ;recommond threhold 5 batch
+    AUTO
+};
+
+enum class KvUpdateAlgo_e : int32_t { V_CACHE_MOVE = 0, K_CACHE_ROPE = 1 };
+
+enum class KvDataType_e : int32_t { BF16 = 0, S8 = 1, INVALID };
+
+#define D0 16
+// kv_cache memory layout:
+//  batch=1: K&V cache [layer,head,1,          max_seq_len/d0,head_dim/d2,d0,d2]
+//  batch>1: k_cache   [layer,head,batch,      max_seq_len/d0,head_dim/d2,d0,d2]
+//           v_cavhe   [layer,head,max_seq_len,batch/d0,      head_dim/d2,d0,d2]
+//  k_cache and v_cache have completely continous memory layout
+struct kv_update_config_p16_s {
+    int64_t base_addr;     // k or v_cache base addr,to compute offset
+    int64_t k_head_stride; // k_cache: size of each head, reduce mul [max_batch,max_seq/d0,head_dim/d2,d0,d2]*byte_size
+    int64_t v_max_seq_len_stride; // v_cache: size of each max_seq_len,
+                                  // reduce mul [max_batch/d0,head_dim/d2,d0,d2]*byte_size
+    int64_t v_head_stride;        // v_cache: size of each head,
+                                  // reduce mul [max_seq_len,max_batch/d0,head_dim/d2,d0,d2]*byte_size
+    KvUpdateAlgo_e kv_update_algo;
+    KvBatchMode_e data_mv_mode;
+
+    int32_t model_batch_num; // to specify one batch or mul batch memory of v_cache,1-onebatch;16-16batch
+    int32_t n_layer;         // all layer
+    int32_t n_head;          // all head
+    int32_t n_batch;         // only valid batch
+    int32_t n_tokens_len;    // valid n_seq_len
+
+    int32_t head_dim_div_d2; // assert head_dim_div_d2 % 2 == 0
+
+    KvDataType_e sin_cos_table_data_type; // support bf16  first; 0-bf16
+    int16_t *sin_table;
+    int16_t *cos_table;
+
+    KvDataType_e kv_data_type; // support bf16 first for k_cache, bf16 s8 for v_cache, 0-bf16,1-s8
+    KvCellAddr_s *addrs;       // [n_batch]
+
+    kv_update_config_p16_s() = default;
+    ~kv_update_config_p16_s() { free(addrs); }
+
+    kv_update_config_p16_s(const kv_update_config_p16_s &) = delete;
+    kv_update_config_p16_s &operator=(const kv_update_config_p16_s &) = delete;
+    kv_update_config_p16_s(kv_update_config_p16_s &&) = default;
+    kv_update_config_p16_s &operator=(kv_update_config_p16_s &&) = default;
+};
+
+struct KvHwBatch {
+    KvHwBatch(int32_t batch);
+    KvSeqId seq_id;
+    int32_t batch_id;
+    size_t seq_len;
+};
+
+struct KvSeqInfo {
+    KvSeqId seq_id;
+    size_t seq_len;
+};
+
+struct CalrtSeqShift {
+    KvSeqId seq_id;
+    KvPos dst_pos;
+    KvPos src_pos;
+    KvPos len;
+};
+
+class CALRT_API KvManager {
+  public:
+    KvManager(calrt::VirtualDevice* vdev, calrt::Calbin* calbin);
+
+    bool canShift();
+
+    // check if we have avaliable hardware resource
+    bool canAllocate(size_t seq_num);
+    void Allocate(KvSeqInfo &seq_info);
+
+    void Free(KvSeqId seq_id);
+
+    void RemoveTokensAtEnd(KvSeqId seq_id, size_t n_tokens);
+
+    bool isExist(KvSeqId seq_id);
+    // keep track of kv state
+    bool Apply(KvSeqInfo &seq_info);
+
+    // if batch_mode == ALL, only seqs[0] will be used
+    void DoShift(KvBatchMode_e batch_mode, std::vector<CalrtSeqShift> &seqs);
+
+    KvPos GetSeqLen(KvSeqId seq_id) const;
+
+    void Clear();
+
+    KvPos seqPosMin(KvSeqId seq_id) const;
+    KvPos seqPosMax(KvSeqId seq_id) const;
+
+  private:
+    VirtualDevice* vdev;
+    Calbin* calbin;
+    CalbinLLM_s llm_spec;
+
+    std::deque<std::unique_ptr<KvHwBatch>> free_hw_batch;
+
+    std::unordered_map<KvSeqId, std::unique_ptr<KvHwBatch>> seq_to_hw_batch;
+
+    std::vector<float> inv_freq;
+    std::unordered_map<int, std::vector<int16_t>> sin_cache; // bf16
+    std::unordered_map<int, std::vector<int16_t>> cos_cache;
+
+    template <float (*tri)(float)> int16_t *get_table(std::unordered_map<int, std::vector<int16_t>> &cache, int offset);
+
+    kv_update_config_p16_s build_kv_config(KvUpdateAlgo_e k_or_v, KvBatchMode_e batch_mode, int32_t max_batch,
+                                           int32_t valid_batch, std::vector<CalrtSeqShift> &seqs);
+};
+
+// std::unique_ptr<KvManager> kv_manager_init_from_spec(const CalbinLLM_s &calbin_llm);
+
+static int16_t float_to_bfloat16(float f) {
+    uint32_t float_bits;
+    std::memcpy(&float_bits, &f, sizeof(float)); // Copy float bits to uint32_t
+
+    // Bfloat16 is essentially the upper 16 bits of the float
+    // with the lower 16 bits of the mantissa truncated.
+    int16_t bfloat16_bits = static_cast<int16_t>(float_bits >> 16);
+
+    return bfloat16_bits;
+}
+
+template <float (*tri)(float)>
+int16_t *KvManager::get_table(std::unordered_map<int, std::vector<int16_t>> &cache, int offset) {
+    auto it = cache.find(offset);
+
+    if (it != cache.end()) {
+        return it->second.data();
+    }
+
+    std::vector<int16_t> table(inv_freq.size() * 2);
+
+    for (size_t i = 0; i < inv_freq.size(); ++i) {
+        float theta = static_cast<float>(-offset) * inv_freq[i]; // use negative pos because we use this for kv shift
+        float val = tri(theta);
+
+        int16_t bf16 = float_to_bfloat16(val);
+
+        table[i] = bf16;
+        table[inv_freq.size() + i] = bf16;
+    }
+
+    auto [ins_it, _] = cache.emplace(offset, std::move(table));
+    return ins_it->second.data();
+}
+
+} // namespace calrt

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_platform.h

@@ -0,0 +1,44 @@
+#pragma once
+
+// ------------------------------
+// CALRT_STATIC: building/using static library
+// CALRT_BUILD_DLL: building shared library (the .dll/.so itself)
+// otherwise: using shared library
+// ------------------------------
+
+#if defined(CALRT_STATIC)
+  #define CALRT_API
+  #define CALRT_LOCAL
+#else
+  #if defined(_WIN32) || defined(__CYGWIN__)
+    #if defined(CALRT_BUILD_DLL)
+      #define CALRT_API __declspec(dllexport)
+    #else
+      #define CALRT_API __declspec(dllimport)
+    #endif
+    #define CALRT_LOCAL
+  #else
+    // GCC/Clang
+    #define CALRT_API   __attribute__((visibility("default")))
+    #define CALRT_LOCAL __attribute__((visibility("hidden")))
+  #endif
+#endif
+
+// Optional: force inline / noinline helpers for ABI stability
+#if defined(_MSC_VER)
+  #define CALRT_NOINLINE __declspec(noinline)
+#else
+  #define CALRT_NOINLINE __attribute__((noinline))
+#endif
+
+// C API helpers
+#ifdef __cplusplus
+  #define CALRT_EXTERN_C extern "C"
+#else
+  #define CALRT_EXTERN_C extern
+#endif
+
+#ifndef LOGURU_EXPORT
+	// Define to your project's export declaration if needed for use in a shared library.
+	#define LOGURU_EXPORT CALRT_API
+#endif

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_utils.h

@@ -0,0 +1,354 @@
+/**
+ * @file calrt_utils.h
+ * @author your name (you@domain.com)
+ * @brief 
+ * @version 0.1
+ * @date 2024-11-11
+ * 
+ * @copyright Copyright (c) 2024
+ * 
+ */
+
+#pragma once
+
+#include "calrt_platform.h"
+
+#include <cstdint>
+#include <vector>
+#include <string>
+#include <cstring>
+#include <unordered_map>
+
+#include "calrt_error.h"
+
+namespace calrt {
+
+enum CALRT_API PrimitiveType 
+{
+  U1=0,     //1bit
+  PRED=1,   //bool, int8
+  U8=2,
+  S8=3,
+  FP8_143=4, // exp_bias = 8; has subnormal, no inf/nan. when biased_exp=0, it is fixed explained as unbiased_exp=-7;
+  FP8_152=5, // exp_bias = 16; has subnormal, no inf/nan.  when biased_exp=0, it is fixed explained as unbiased_exp=-15;
+  U16=6,
+  S16=7,    //16b
+  BF16=8,   //16b
+  U32=9,    //32b
+  S32=10,
+  F32=11,
+  U4=12,
+  S4=13,
+  F35=14,
+  U64=15,   // not support
+  S64=16,   // u64和s64需要保留，仅做类型转换，因为pytorch要求tensor做index时必须为long或byte或bool。
+
+  TF32=17,//19b    // not support
+  F16=18,
+  F64=19,
+  C64,
+  C128,
+
+  TUPLE,//0
+  TOKEN,//32b
+  OPAQUE_TYPE,//32b
+  INVALID, //0b
+  TYPE_SIZE
+};
+
+PrimitiveType PrimitiveTypeFromInt(const int32_t& type);
+
+std::string PrimitiveTypeToString(const PrimitiveType& type);
+std::string PrimitiveTypeToString(const int32_t& type);
+
+CALRT_API int32_t PrimitiveTypeBitSize(const PrimitiveType& type);
+
+struct CALRT_API CalbinTensorInfo_s {
+    std::string             name;
+    std::vector<uint64_t>    shape;
+    PrimitiveType           dataType;
+    int64_t                 devAddr0 = -1;   // ping
+    int64_t                 devAddr1 = -1;   // pong
+    int64_t                 size = -1;
+};
+
+enum CalbinSectionType_e {
+    CALBIN_SECTION_NULL     = 0,
+    CALBIN_SECTION_CPU_CMD  ,
+    CALBIN_SECTION_CPU_SO   ,
+    CALBIN_SECTION_CCU_ELF  ,
+    CALBIN_SECTION_RODATA   ,
+    CALBIN_SECTION_IBUF     ,
+    CALBIN_SECTION_OBUF     ,
+    CALBIN_SECTION_MEMRSVD  ,
+    CALBIN_SECTION_KV       ,
+    CALBIN_SECTION_CSR      ,
+    CALBIN_SECTION_NUM
+};
+
+constexpr const char * CalbinSecTypeToString(CalbinSectionType_e e)
+{
+    switch (e)
+    {
+        case CALBIN_SECTION_CPU_CMD: return "CPU command";
+        case CALBIN_SECTION_CCU_ELF: return "CCU ELF";
+        case CALBIN_SECTION_CPU_SO: return ".so file";
+        case CALBIN_SECTION_RODATA: return "parameter";
+        case CALBIN_SECTION_IBUF: return "input buffer";
+        case CALBIN_SECTION_OBUF: return "output buffer";
+        case CALBIN_SECTION_MEMRSVD: return "reserved memory";
+        case CALBIN_SECTION_KV: return "kv cache";
+        case CALBIN_SECTION_CSR: return "csr";
+        default: return "null";
+    }
+}
+
+enum CalbinSectionPlace_e{
+    DRAM = 0,
+    SRAM,
+    UNKNOWN
+};
+
+/**
+ * @brief 
+ * @note CalbinSectionType_e == CPU_SO. only need offset(used for prgIdx) devAddr, size,
+ */
+struct CALRT_API CalbinSection_s {
+    CalbinSectionPlace_e            place       = UNKNOWN;
+    CalbinSectionType_e             type        = CALBIN_SECTION_NULL; // cpu txt, ccu elf, data
+    std::string                     name        = "";
+    std::string                     filePath    = ""; // must include elf/cpu file path
+    int64_t                         offset      = -1; // [progbit & .bss] entry address
+    int64_t                         devAddr     = -1;    // -1 means calrt set memory, for memory and program
+    int64_t                         size        = -1;
+    int64_t                         progEntry   = -1; // program header entry address
+
+    std::vector<uint32_t>           chipMask;     // mask2, mask1, mask0, small-endian
+    std::vector<CalbinTensorInfo_s>   tensorInfos;        // for i/o buf
+
+    bool operator<(const CalbinSection_s& other) const;
+    bool operator==(const CalbinSection_s& other) const;
+    bool operator!=(const CalbinSection_s& other) const;
+    friend std::ostream& operator<< (std::ostream& os, const CalbinSection_s&);
+};
+
+struct CALRT_API KV_Cache_s
+{
+    PrimitiveType dtype = INVALID;
+    PrimitiveType sin_cos_table_data_type = INVALID;
+    uint32_t layer;
+    uint64_t kv_base[2]; //[0] k base address, [1] v base address
+    uint64_t size;
+    uint32_t n_head;
+    uint32_t n_dim;
+
+    operator bool () const noexcept
+    {
+        return dtype != INVALID;
+    }
+};
+
+struct CALRT_API CalbinLLM_s
+{
+    uint32_t max_batch_size = 0; // 16
+    uint32_t max_seq_len = 0;
+
+    KV_Cache_s kv_cache;
+    operator bool () const noexcept
+    {
+        return max_batch_size != 0 && max_seq_len != 0;
+    }
+};
+
+enum ModelChipMode_e
+{
+    SCHIP=0,
+    MCHIP=1,
+    UNDEFINED_CHIP_MODE
+};
+
+struct CALRT_API CalbinModel {
+    enum CalDevAccType_e {
+        CAL_DEV_ACC_NULL    = 0,
+        CAL_DEV_ACC_CPU     = 1,
+        CAL_DEV_ACC_CCU     = 2,
+        CAL_DEV_ACC_CPU_CCU = 3
+    };
+
+    bool                    isConfigured = false;
+    CalDevAccType_e         devAccType;
+    ModelChipMode_e         chipMode = UNDEFINED_CHIP_MODE;
+    std::string             modelType;
+    std::string             modelArch;
+    std::string             modelName;
+
+    std::unordered_map<CalbinSectionType_e, std::vector<CalbinSection_s>> sections; //idx follow above
+    std::vector<int32_t>    tarChipN;
+};
+
+struct CALRT_API CalrtCalbin {
+    uint64_t name; // md5 value or hash value
+    std::vector<CalbinModel>  models;
+};
+
+enum CalrtBufferDirection_e{
+    HostToDevice = 0,
+    DeviceToHost = 1,
+};
+
+enum class PowerMode : uint8_t
+{
+    BALANCE = 1,
+    HIGH_PERFORMANCE = 2
+};
+
+class CALRT_API CalrtTensor {
+
+public:
+    CalrtTensor() = delete;
+
+    CalrtTensor(const std::vector<uint64_t> &shape, PrimitiveType elemType, CalrtBufferDirection_e direction, std::string name = "");
+
+    template<typename T>
+    CalrtError_e CheckTensorByDataSize(const std::vector<T> &datas_or_shape, PrimitiveType dtype)
+    {
+
+        bool success = mElemBitSize == PrimitiveTypeBitSize(dtype);
+        if(!success) return CalrtErrorInvalidDataType;
+        success = success && ((int64_t)datas_or_shape.size() == mElemSize);
+
+        if(!success) return CalrtErrorInvalidDataShape;
+
+        return CalrtSuccess;
+    }
+
+    CalrtError_e CheckTensor(const std::vector<uint64_t> &shape, PrimitiveType dtype);
+
+    /**
+     * @brief safe to copy date to tensor in byte
+     * @note for special custom data type, Using GetDataPtr() to manuelly fill data into tensor.
+     * 
+     * @tparam T 
+     * @param datas input data
+     * @param dtype data type
+     */
+    template<typename T>
+    void Fill(std::vector<T> &datas, PrimitiveType dtype)
+    {
+        FillImpl(static_cast<void*>(datas.data()), CheckTensorByDataSize(datas, dtype));
+    }
+
+    /**
+     * @brief map host buffer address to device address
+     * @note unsafety for non-tensor check
+     * 
+     * @param src 
+     */
+    void MapBuf(void *src);
+
+    void UnMapBuf();
+
+    /**
+     * @brief Get the Data Ptr object. User must guarantee a safety copy action
+     * 
+     * @return char* 
+     */
+    uint8_t* GetDataPtr();
+
+    PrimitiveType Type() {return mElemType;}
+    const std::vector<uint64_t> &Shape() {return mShape;}
+
+    /**
+     * @brief Get the current tensor byte size
+     * 
+     * @return const int64_t 
+     */
+    uint64_t ByteSize() {return mByteSize;}
+    
+    /**
+     * @brief Get the number of element from the current tensor based on dtype
+     * 
+     * @return const int64_t 
+     */
+    uint64_t Size() {return mElemSize;}
+
+    const std::string &Name() {return mName;}
+
+    CalrtBufferDirection_e Direction();
+
+    CalrtError_e SliceTensor(uint64_t offset, uint64_t size);
+    void UndoSlice();
+    uint64_t Offset();
+    uint64_t TransSize();
+
+    friend std::ostream& operator<<(std::ostream& os, const CalrtTensor& obj);
+
+    private:
+        void FillImpl(void* src, CalrtError_e err);
+        
+        std::vector<uint64_t> mShape;
+        PrimitiveType mElemType;    // tensor data elem type
+        CalrtBufferDirection_e mDirection;
+        std::string mName;
+        uint32_t mElemBitSize;   // each elem bit size
+        uint64_t mElemSize;      // elem size
+        uint64_t mByteSize;      // momory byte size
+        uint64_t mOffset;
+        uint64_t mTransDataSize;
+        std::vector<uint8_t> mData;
+        uint8_t *mRawPtr = nullptr;
+};
+
+struct CalrtDevBuf_s {
+    int64_t addr0 = -1;
+    int64_t addr1 = -1;
+    int64_t size = -1;
+
+    friend std::ostream& operator<<(std::ostream& os, const CalrtDevBuf_s& obj);
+};
+
+struct CalrtBufferInfo_s{
+    CalrtBufferDirection_e direction;
+    std::string name;
+    std::vector<CalbinTensorInfo_s> tensorInfos;
+    std::vector< std::pair<std::string, uint32_t>> csrTable;    
+};
+
+enum class TaskType_e : uint32_t
+{
+    TASK_PING = 0,
+    TASK_PONG,
+    UNDEFINED_TASK
+};
+
+enum class ChipArch_e : uint32_t
+{
+    SINGLE_CHIP_TASK = 0,
+    MULTIPLE_CHIP_TASK,
+    UNDEFINED
+};
+
+struct TaskInfo_s
+{
+    TaskType_e m_taskType;
+    uint32_t m_jobId;
+    uint32_t m_enableCcu;
+    ChipArch_e m_taskChipArch;
+    uint32_t m_syncRegIdx;
+    uint32_t m_syncExpectedVal;
+};
+
+// struct CalrtVersion
+// {
+//     uint32_t major; // API or ABI change. E.G. remove/change function, user may need to adapt their code
+//     uint32_t minor; // added new feature
+//     uint32_t patch; // fix bug
+// };
+
+CALRT_API const char *calrt_version();
+
+CALRT_API void SetFullDebug(bool enable);
+
+CALRT_API bool isFullDebug();
+
+}

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/calrt_vdevice.h

@@ -0,0 +1,114 @@
+/**
+ * @file calrt_vdevice.h
+ * @brief temporary only support one physical device.
+ * @version 0.1
+ * @date 2024-11-11
+ * 
+ * @copyright Copyright (c) 2024
+ * 
+ */
+
+#pragma once
+
+#include <memory>
+#include <atomic>
+
+#include "calrt_device.h"
+#include "calrt_buffer.h"
+
+namespace calrt
+{
+    //class CalrtDeviceHandle; //  device resource manage
+    //struct CalrtDeviceStatus;//  record device status
+    //class CalDeviceParams; // the device params;
+
+    class CALRT_API VirtualDevice
+    {
+    public:
+        ~VirtualDevice();
+        
+        static std::unique_ptr<VirtualDevice> CreateVDevice();//create device using default params;
+        static std::unique_ptr<VirtualDevice> CreateVDevice(CalrtDeviceType_e type);
+
+        CalrtDevice* GetDevice() { return m_phyDevices.get(); }
+        const char *GetDeviceInfo(size_t idx) {return m_phyDevices->GetDevInfo().c_str();}
+        // std::size_t GetPhyDeviceNum() { return m_phyDevices.size(); }
+
+        /**
+         * @brief 
+         * 
+         * @param model 
+         * @param inputBuffer 
+         * @param outputBuffer 
+         * @param forceEngineMode default: -1, automatically run engine mode. 0: force ping. 1: force pong
+         */
+        void SubmitJob(CalbinModel* model, CalrtInputBuf* inputBuffer, CalrtOutputBuf* outputBuffer, TaskType_e forceEngineMode = TaskType_e::UNDEFINED_TASK);
+        void Shutdown();
+        void ReportDeviceInfo();
+        void EnableTraceDevice(bool enable) {m_phyDevices->EnableTraceDevice(enable);}
+        CalrtError_e Status();
+        void EnableParallelMode(bool switch_on);
+
+        /**
+         * @brief read device memory from selected chip
+         * 
+         * @param srcAddr 
+         * @param dst 
+         * @param size 
+         * @param chipId chip id, default 0
+         * @return CalrtError_e 
+         */
+        CalrtError_e ReadMem(void *srcAddr, uint64_t dst, uint64_t size, uint32_t chipId = 0);
+
+        /**
+         * @brief write data to selected chip's memory
+         * 
+         * @param srcAddr 
+         * @param dst 
+         * @param size 
+         * @param chipId chip id, default 0
+         * @return CalrtError_e 
+         */
+        CalrtError_e WriteMem(void *srcAddr, uint64_t dst, uint64_t size, uint32_t chipId = 0);
+        
+        /**
+         * @brief reset CCU
+         * 
+         */
+        void ResetCCU();
+
+        /**
+         * @brief reset deployed calbin on the current device
+         * 
+         */
+        void ResetConfiguration();
+
+        /**
+         * @brief reset device
+         * 
+         */
+        void Reset();
+
+        /**
+         * @brief release device. for handle sigabrt, sigint signal
+         * 
+         */
+        void Release();
+
+        CalrtDeviceType_e Type() noexcept;
+
+        VirtualDevice(const VirtualDevice &) = delete;
+        VirtualDevice &operator=(const VirtualDevice&) = delete;
+        VirtualDevice(VirtualDevice &&) = delete;
+        VirtualDevice &operator=(VirtualDevice &&other) = delete;
+  
+    private:
+
+        VirtualDevice();
+
+        class CalrtJobEngine;
+        std::unique_ptr<CalrtJobEngine> m_dispatchEngine;
+        std::atomic<bool> m_shutdown;
+        std::shared_ptr<CalrtDevice> m_phyDevices;
+    };
+}

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/elf_parser.hpp

@@ -0,0 +1,82 @@
+#pragma once
+
+#include <cstring>
+#include <cstdint>
+#include <vector>
+#include <map>
+
+namespace calc_efl
+{
+    enum class LoadFlag : uint32_t
+    {
+        DO_NOT_LOAD = 0,
+        BY_SECTION = 1,
+        BY_PT_LOAD
+    };
+
+    struct Parsed_Elf_Section_s
+    {
+        char name[64];
+        uint32_t type;
+        uint64_t size; // size after padding if in need
+        uint64_t address;
+        uint64_t offset = 0; // indicate the raw efficient data index in image
+        std::vector<uint8_t> data;
+    };
+
+    // struct Parsed_shared_ph_s
+    // {
+    //     uint32_t p_type;
+    //     uint32_t p_flgas;
+    //     uint64_t p_vaddr;
+    //     uint64_t p_filesz;
+    // };
+
+    struct Parsed_Elf_s
+    {
+        uint16_t id;
+        uint16_t e_type;
+        uint64_t e_entry;
+        uint64_t progBitSize = 0; // only record max progbit type siz (cross all elf under the same model)
+        uint64_t f_addr = 0; // indicate the first valid section address. For relocation usage
+        char path[256];
+        char postfix[128]; // if is same chip ccu
+        // std::map<uint64_t, Parsed_shared_ph_s> phs;
+        std::map<uint64_t, Parsed_Elf_Section_s> sections; // [address, section]
+
+        bool operator ==(const Parsed_Elf_s &other) const
+        {
+            bool isSamePost = std::strcmp(postfix, other.postfix) == 0;
+            bool isSamePath = std::strcmp(path, other.path) == 0;
+            return (isSamePost && isSamePath);
+        }
+    };
+
+    /**
+     * @brief intelligent parse method
+     * 
+     * @param path ELF path
+     * @return std::pair<uint64_t, std::vector<uint8_t>> entry addr and datas
+     */
+    std::pair<uint64_t, std::vector<uint8_t>> ParseElf_Intell(const char *path);
+    
+    /**
+     * @brief naively parse metho
+     * 
+     * @param path elf path
+     * @param isRT is it for calrt purpose (default: false)
+     * @return Parsed_Elf_s 
+     */
+    Parsed_Elf_s ParseElf_Naive(const char *path, LoadFlag lFlag = LoadFlag::BY_SECTION);
+
+    Parsed_Elf_s ParseElf_by_PT_LOAD(const char *path);
+
+    /**
+     * @brief relocate elf
+     * 
+     * @param elf_mirror data struct get from ParseElf
+     * @param path elf file path
+     * @param loadBase new allocated address
+     */
+    void RelocateSection(uint8_t *imageElf, const char *path, uint64_t loadBase, const Parsed_Elf_s &parsed_elf);
+}

source/baseline-expanded/calrt-sdk-0.7.6/usr/local/include/calrt/protocol.hpp

@@ -0,0 +1,135 @@
+/**
+ * @file protocol.hpp
+ * @brief IPC protocol
+ * 
+ * payload layout: [direction | RegionKey | Region]
+ * IPC data share protocol layout : [ShMemhdr | large data]
+ * Overall layout: [Payload | ShMemhdr | large data]
+ * 
+ * device management layout: [ShmHdr | ShmCtx ...]
+ * 
+ * @version 0.1
+ * @date 2025-10-29
+ * 
+ * @copyright Copyright (c) 2025
+ * 
+ */
+
+#pragma once
+
+// #include <sys/mman.h> // For mmap and munmap
+#include <cstring>
+#include <pthread.h>
+#include <cstdint>
+#include <atomic>
+
+// #ifdef __cplusplus
+//     #include <atomic>
+//     #define ALIGNAS(x) alignas(x)
+//     #define ATOMIC_UINT32 std::atomic<uint32_t>
+// #else
+//     #include <stdatomic.h>
+//     #define ALIGNAS(x) _Alignas(x)
+//     #define ATOMIC_UINT32 _Atomic uint32_t
+// #endif
+
+namespace calrt
+{
+    extern const char* kSockPath;
+    extern const char* kShMemCreatePath;
+    extern const char* kShMemRepairPath;
+
+    // inline constexpr const char* kSockPath = "/home/patrickyang/uds_epoll_demo.sock";
+    inline constexpr const char* kShMem = "/shm_calculet_device";
+    // inline constexpr const char* kShMemCreatePath = "/run/create_lck.lock";
+    // inline constexpr const char* kShMemRepairPath = "/run/repair.lock";
+    // inline constexpr const char* kShMemRepairPathBak = "/tmp/calrt/repair.lock";
+
+    // -------------------------------------- IPC large data transfer --------------------------------------
+    // 
+    /**
+     * @brief each region key coresponse to one req or many if there are related. just tell the offset and size with same region id.
+     * 
+     */
+    struct RegionKey
+    {
+        int32_t id;
+        bool operator==(const RegionKey &other) const noexcept
+        {
+            return id == other.id;
+        }
+    };
+
+    struct Region
+    {
+        int32_t fd = -1; // don't init in client side
+        int32_t prot; // = PROT_READ | PROT_WRITE; // protection flag for kernel. tell how process can access the mapped memory region
+        int32_t flag; // = MAP_SHARED;
+        std::atomic<uint32_t> counter{0}; // counter if this region is still accessed
+        uint64_t offset = 0; // temp represent memory address.
+        uint64_t size = 0;
+        void *map = nullptr;
+    };
+
+    struct ShMemhdr
+    {
+        std::atomic<uint32_t> ready = 0; // 0: not ready, 1: ready
+    };
+
+    struct Payload
+    {
+        uint8_t direction; // 0: host-to-device. 1: device-to-host
+        RegionKey key;
+        Region region;
+    };
+    // -------------------------------------- end --------------------------------------
+
+    // -------------------------------------- helper --------------------------------------
+    inline uint64_t now_ns()
+    {
+        struct timespec ts;
+        clock_gettime(CLOCK_MONOTONIC, &ts);
+        return static_cast<uint64_t>(ts.tv_sec) * 1000000000ull + ts.tv_nsec;
+    }
+
+    template<typename T>
+    void encode_impl(std::vector<unsigned char> &meta, T data, size_t len = 0)
+    {
+        size_t n = meta.size();
+        size_t tSize = len == 0? sizeof(T) : len;
+        meta.resize(n+tSize);
+        std::memcpy(&meta[n], &data, tSize);
+    }
+
+    /**
+     * @brief [id | prot | flag | size]
+     * 
+     * @param meta 
+     * @param regionK 
+     * @param region 
+     */
+    inline void encode_socket(std::vector<unsigned char> &meta, Payload &msgHdr)
+    {
+        encode_impl(meta, msgHdr.direction);
+        encode_impl(meta, msgHdr.key.id);
+        encode_impl(meta, msgHdr.region.prot);
+        encode_impl(meta, msgHdr.region.flag);
+        encode_impl(meta, msgHdr.region.size);
+        encode_impl(meta, msgHdr.region.offset);
+    }
+
+    inline void decode_socket(const unsigned char* meta, Payload &out)
+    {        
+        std::memcpy(&out.direction, meta, sizeof(out.direction));
+        size_t stride = sizeof(out.direction);
+        std::memcpy(&out.key.id, meta+stride, sizeof(out.key.id));
+        stride += sizeof(out.key.id);
+        std::memcpy(&out.region.prot, meta+stride, sizeof(out.region.prot));
+        stride += sizeof(out.region.prot);
+        std::memcpy(&out.region.flag, meta+stride, sizeof(out.region.flag));
+        stride += sizeof(out.region.flag);
+        std::memcpy(&out.region.size, meta+stride, sizeof(out.region.size));
+        stride += sizeof(out.region.size);
+        std::memcpy(&out.region.offset, meta+stride, sizeof(out.region.offset));
+    }
+} //namespace calrt