966 lines
34 KiB
C
966 lines
34 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/**
|
|
* Copyright (c) 2024-2025 CALCULET Technologies Ltd. All rights reserved.
|
|
*/
|
|
|
|
#include "dma.h"
|
|
#include "board_mgr.h"
|
|
#include "utils.h"
|
|
|
|
#include <linux/sched.h>
|
|
#include <linux/version.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/uaccess.h>
|
|
#include <asm/cacheflush.h>
|
|
|
|
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
|
|
#include <linux/dma-map-ops.h>
|
|
#else
|
|
#include <linux/dma-mapping.h>
|
|
#endif
|
|
|
|
static int calculet_set_dma_mask(struct device *dev)
|
|
{
|
|
int err = -EINVAL;
|
|
/* Check and configure DMA length */
|
|
if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))) {
|
|
calculet_dev_log(CALCULET_LOG_INFO, dev, "Enabled 64 bit dma");
|
|
} else if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)))) {
|
|
calculet_dev_log(CALCULET_LOG_DEBUG, dev, "Enabled 48 bit dma");
|
|
} else if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)))) {
|
|
calculet_dev_log(CALCULET_LOG_DEBUG, dev, "Enabled 40 bit dma");
|
|
} else if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)))) {
|
|
calculet_dev_log(CALCULET_LOG_DEBUG, dev, "Enabled 36 bit dma");
|
|
} else if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)))) {
|
|
calculet_dev_log(CALCULET_LOG_INFO, dev, "Enabled 32 bit dma");
|
|
} else {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "Error enabling dma %d", err);
|
|
return err;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
// DMA通道状态管理函数（简化，减少不必要的锁）
|
|
void calculet_dma_channel_set_state(struct calculet_dma_channel *channel,
|
|
enum calculet_dma_channel_state state)
|
|
{
|
|
if (channel && channel->state != state) {
|
|
calculet_log(CALCULET_LOG_DEBUG, "DMA channel %d state change: %d -> %d",
|
|
channel->channel_index, channel->state, state);
|
|
WRITE_ONCE(channel->state, state);
|
|
channel->last_used = ktime_get();
|
|
}
|
|
}
|
|
|
|
enum calculet_dma_channel_state calculet_dma_channel_get_state(struct calculet_dma_channel *channel)
|
|
{
|
|
return channel ? READ_ONCE(channel->state) : DMA_CHANNEL_ERROR;
|
|
}
|
|
|
|
bool calculet_dma_channel_is_available(struct calculet_dma_channel *channel)
|
|
{
|
|
if (!channel) return false;
|
|
|
|
enum calculet_dma_channel_state state = calculet_dma_channel_get_state(channel);
|
|
return (state == DMA_CHANNEL_IDLE);
|
|
}
|
|
|
|
// 通道分配和释放（简化，使用原子操作）
|
|
int calculet_dma_allocate_channel(struct calculet_dma_controller *controller,
|
|
enum calculet_transfer_direction direction,
|
|
int *channel_id)
|
|
{
|
|
unsigned long *bitmap;
|
|
struct calculet_dma_channel *channels;
|
|
int max_channels;
|
|
int id;
|
|
|
|
if (!controller || !channel_id) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (direction == TRANSFER_H2C_DIRECTION) {
|
|
bitmap = &controller->read_channels_bitmap;
|
|
channels = controller->dma_rdch_s;
|
|
max_channels = CALCULET_READ_CHANNELS;
|
|
} else {
|
|
bitmap = &controller->write_channels_bitmap;
|
|
channels = controller->dma_wrch_s;
|
|
max_channels = CALCULET_WRITE_CHANNELS;
|
|
}
|
|
|
|
// 查找可用通道（无锁版本）
|
|
do {
|
|
id = find_first_zero_bit(bitmap, max_channels);
|
|
if (id >= max_channels) {
|
|
return -EBUSY;
|
|
}
|
|
|
|
// 检查通道状态
|
|
if (!calculet_dma_channel_is_available(&channels[id])) {
|
|
continue;
|
|
}
|
|
|
|
} while (test_and_set_bit(id, bitmap));
|
|
|
|
calculet_dma_channel_set_state(&channels[id], DMA_CHANNEL_BUSY);
|
|
*channel_id = id;
|
|
|
|
calculet_log(CALCULET_LOG_DEBUG, "Allocated %s channel %d",
|
|
direction == TRANSFER_H2C_DIRECTION ? "H2C" : "C2H", id);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void calculet_dma_release_channel(struct calculet_dma_controller *controller,
|
|
enum calculet_transfer_direction direction,
|
|
int channel_id)
|
|
{
|
|
unsigned long *bitmap;
|
|
struct calculet_dma_channel *channels;
|
|
int max_channels;
|
|
|
|
if (!controller) {
|
|
return;
|
|
}
|
|
|
|
if (direction == TRANSFER_H2C_DIRECTION) {
|
|
bitmap = &controller->read_channels_bitmap;
|
|
channels = controller->dma_rdch_s;
|
|
max_channels = CALCULET_READ_CHANNELS;
|
|
} else {
|
|
bitmap = &controller->write_channels_bitmap;
|
|
channels = controller->dma_wrch_s;
|
|
max_channels = CALCULET_WRITE_CHANNELS;
|
|
}
|
|
|
|
if (channel_id >= 0 && channel_id < max_channels) {
|
|
clear_bit(channel_id, bitmap);
|
|
calculet_dma_channel_set_state(&channels[channel_id], DMA_CHANNEL_IDLE);
|
|
|
|
calculet_log(CALCULET_LOG_DEBUG, "Released %s channel %d",
|
|
direction == TRANSFER_H2C_DIRECTION ? "H2C" : "C2H", channel_id);
|
|
}
|
|
}
|
|
|
|
// 传输上下文管理（简化）
|
|
struct calculet_dma_transfer_context* calculet_dma_create_transfer_context(pid_t pid)
|
|
{
|
|
struct calculet_dma_transfer_context *ctx;
|
|
|
|
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
|
|
if (!ctx) {
|
|
return NULL;
|
|
}
|
|
|
|
init_completion(&ctx->completion);
|
|
ctx->result = 0;
|
|
ctx->start_time = ktime_get();
|
|
ctx->requester_pid = pid;
|
|
atomic_set(&ctx->ref_count, 1);
|
|
|
|
return ctx;
|
|
}
|
|
|
|
void calculet_dma_destroy_transfer_context(struct calculet_dma_transfer_context *ctx)
|
|
{
|
|
if (ctx) {
|
|
kfree(ctx);
|
|
}
|
|
}
|
|
|
|
void calculet_dma_transfer_context_get(struct calculet_dma_transfer_context *ctx)
|
|
{
|
|
if (ctx) {
|
|
atomic_inc(&ctx->ref_count);
|
|
}
|
|
}
|
|
|
|
void calculet_dma_transfer_context_put(struct calculet_dma_transfer_context *ctx)
|
|
{
|
|
if (ctx && atomic_dec_and_test(&ctx->ref_count)) {
|
|
calculet_dma_destroy_transfer_context(ctx);
|
|
}
|
|
}
|
|
|
|
int calculet_dma_controller_init(struct calculet_dma_controller *controller, struct device *dev, bool poll_en)
|
|
{
|
|
int i;
|
|
int err = 0;
|
|
struct calculet_dma_poll *poll;
|
|
|
|
if (!dev) {
|
|
calculet_log(CALCULET_LOG_ERROR, "Device pointer is NULL");
|
|
return -EINVAL;
|
|
}
|
|
|
|
controller->dev = dev;
|
|
controller->controller_enabled = false;
|
|
|
|
// 初始化控制器级别的锁和状态
|
|
mutex_init(&controller->controller_mutex);
|
|
mutex_init(&controller->allocation_mutex);
|
|
atomic_set(&controller->active_transfers, 0);
|
|
|
|
// 初始化通道分配位图
|
|
controller->read_channels_bitmap = 0;
|
|
controller->write_channels_bitmap = 0;
|
|
|
|
/* Check and configure DMA length */
|
|
err = calculet_set_dma_mask(dev);
|
|
if (0 > err) {
|
|
return err;
|
|
}
|
|
|
|
if (get_dma_ops(controller->dev)) {
|
|
calculet_dev_log(CALCULET_LOG_DEBUG, controller->dev, "Using specialized dma_ops=%ps", get_dma_ops(controller->dev));
|
|
}
|
|
else {
|
|
calculet_dev_log(CALCULET_LOG_WARN, dev, "No specialized dma_ops found");
|
|
}
|
|
|
|
// 初始化读通道（简化初始化）
|
|
for(i = 0; i < CALCULET_READ_CHANNELS; i++) {
|
|
struct calculet_dma_channel *channel = &controller->dma_rdch_s[i];
|
|
|
|
memset(channel, 0, sizeof(*channel));
|
|
channel->channel_index = i;
|
|
channel->state = DMA_CHANNEL_IDLE;
|
|
|
|
spin_lock_init(&channel->state_lock);
|
|
mutex_init(&channel->buffer_mutex);
|
|
mutex_init(&channel->transfer_mutex);
|
|
atomic_set(&channel->buffer_users, 0);
|
|
atomic_set(&channel->transfer_finish, 0);
|
|
atomic64_set(&channel->transfer_count, 0);
|
|
atomic64_set(&channel->transfer_bytes, 0);
|
|
init_waitqueue_head(&channel->channel_wq);
|
|
}
|
|
|
|
// 初始化写通道
|
|
for(i = 0; i < CALCULET_WRITE_CHANNELS; i++) {
|
|
struct calculet_dma_channel *channel = &controller->dma_wrch_s[i];
|
|
|
|
memset(channel, 0, sizeof(*channel));
|
|
channel->channel_index = i;
|
|
channel->state = DMA_CHANNEL_IDLE;
|
|
|
|
spin_lock_init(&channel->state_lock);
|
|
mutex_init(&channel->buffer_mutex);
|
|
mutex_init(&channel->transfer_mutex);
|
|
atomic_set(&channel->buffer_users, 0);
|
|
atomic_set(&channel->transfer_finish, 0);
|
|
atomic64_set(&channel->transfer_count, 0);
|
|
atomic64_set(&channel->transfer_bytes, 0);
|
|
init_waitqueue_head(&channel->channel_wq);
|
|
}
|
|
|
|
// 初始化轮询模式
|
|
if(poll_en) {
|
|
/* 分配 DMA Kthread 结构体内存 */
|
|
if(!controller->dma_kthread) {
|
|
controller->dma_kthread = devm_kzalloc(dev, sizeof(struct calculet_dma_kthread), GFP_KERNEL);
|
|
if (!controller->dma_kthread) {
|
|
return -ENOMEM;
|
|
}
|
|
}
|
|
|
|
/* 分配 DMA Poll 结构体内存 */
|
|
if (!controller->dma_kthread->dma_poll) {
|
|
controller->dma_kthread->dma_poll = devm_kzalloc(dev, sizeof(struct calculet_dma_poll), GFP_KERNEL);
|
|
if (!controller->dma_kthread->dma_poll) {
|
|
return -ENOMEM;
|
|
}
|
|
}
|
|
poll = controller->dma_kthread->dma_poll;
|
|
|
|
poll->status_region = dma_alloc_coherent(dev, sizeof(uint32_t), &poll->status_region_phys, GFP_KERNEL);
|
|
if (!poll->status_region) {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to allocate coherent memory for status region");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
spin_lock_init(&poll->poll_lock);
|
|
atomic_set(&poll->poll_active, 0);
|
|
|
|
for (i = 0; i < CALCULET_TOTAL_CHANNELS; i++) {
|
|
poll->channel_data[i] = i + 100;
|
|
}
|
|
|
|
// 初始化内核线程
|
|
atomic_set(&controller->dma_kthread->should_stop, 0);
|
|
init_completion(&controller->dma_kthread->thread_completion);
|
|
}
|
|
|
|
controller->controller_enabled = true;
|
|
|
|
calculet_log(CALCULET_LOG_DEBUG, "DMA controller initialized (poll_mode: %s)",
|
|
poll_en ? "enabled" : "disabled");
|
|
|
|
return 0;
|
|
}
|
|
|
|
void calculet_dma_controller_deinit(struct calculet_dma_controller *controller, bool poll_en)
|
|
{
|
|
int i;
|
|
struct calculet_dma_poll *poll;
|
|
|
|
if (!controller) {
|
|
return;
|
|
}
|
|
|
|
controller->controller_enabled = false;
|
|
|
|
// 等待所有活动传输完成
|
|
while (atomic_read(&controller->active_transfers) > 0) {
|
|
msleep(10);
|
|
}
|
|
|
|
// 清理读通道
|
|
for(i = 0; i < CALCULET_READ_CHANNELS; i++) {
|
|
struct calculet_dma_channel *channel = &controller->dma_rdch_s[i];
|
|
|
|
// 设置通道为禁用状态
|
|
calculet_dma_channel_set_state(channel, DMA_CHANNEL_DISABLED);
|
|
|
|
// 释放缓冲区
|
|
mutex_lock(&channel->buffer_mutex);
|
|
if (channel->kernel_buf) {
|
|
dma_free_coherent(controller->dev, channel->dma_len,
|
|
channel->kernel_buf, channel->dma_handle);
|
|
channel->kernel_buf = NULL;
|
|
channel->dma_len = 0;
|
|
}
|
|
mutex_unlock(&channel->buffer_mutex);
|
|
|
|
// 清理传输上下文
|
|
if (channel->current_transfer) {
|
|
calculet_dma_transfer_context_put(channel->current_transfer);
|
|
channel->current_transfer = NULL;
|
|
}
|
|
}
|
|
|
|
// 清理写通道
|
|
for(i = 0; i < CALCULET_WRITE_CHANNELS; i++) {
|
|
struct calculet_dma_channel *channel = &controller->dma_wrch_s[i];
|
|
|
|
// 设置通道为禁用状态
|
|
calculet_dma_channel_set_state(channel, DMA_CHANNEL_DISABLED);
|
|
|
|
// 释放缓冲区
|
|
mutex_lock(&channel->buffer_mutex);
|
|
if (channel->kernel_buf) {
|
|
dma_free_coherent(controller->dev, channel->dma_len,
|
|
channel->kernel_buf, channel->dma_handle);
|
|
channel->kernel_buf = NULL;
|
|
channel->dma_len = 0;
|
|
}
|
|
mutex_unlock(&channel->buffer_mutex);
|
|
|
|
// 清理传输上下文
|
|
if (channel->current_transfer) {
|
|
calculet_dma_transfer_context_put(channel->current_transfer);
|
|
channel->current_transfer = NULL;
|
|
}
|
|
}
|
|
|
|
// 清理轮询相关资源
|
|
if(poll_en && controller->dma_kthread) {
|
|
poll = controller->dma_kthread->dma_poll;
|
|
if (poll) {
|
|
for (i = 0; i < CALCULET_TOTAL_CHANNELS; i++) {
|
|
poll->channel_data[i] = 0;
|
|
}
|
|
|
|
if (poll->status_region) {
|
|
dma_free_coherent(controller->dev, sizeof(uint32_t),
|
|
poll->status_region, poll->status_region_phys);
|
|
poll->status_region = NULL;
|
|
}
|
|
}
|
|
}
|
|
|
|
calculet_log(CALCULET_LOG_DEBUG, "DMA controller deinitialized");
|
|
}
|
|
|
|
// DMA参数验证
|
|
int calculet_dma_validate_transfer_params(struct calculet_dma_transfer_channels_params *params)
|
|
{
|
|
if (!params) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
// 检查传输方向
|
|
if (params->dma_transfer_direction != TRANSFER_H2C_DIRECTION &&
|
|
params->dma_transfer_direction != TRANSFER_C2H_DIRECTION) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
// 检查通道ID
|
|
if (params->dma_transfer_direction == TRANSFER_H2C_DIRECTION) {
|
|
if (params->dma_channel_id < 0 || params->dma_channel_id >= CALCULET_READ_CHANNELS) {
|
|
return -EINVAL;
|
|
}
|
|
} else {
|
|
if (params->dma_channel_id < 0 || params->dma_channel_id >= CALCULET_WRITE_CHANNELS) {
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
// 检查传输大小
|
|
if (params->dma_channel_size == 0 || params->dma_channel_size > (32 * 1024 * 1024)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
// 检查地址对齐
|
|
if (params->device_address & 0x3) {
|
|
calculet_log(CALCULET_LOG_WARN, "Device address 0x%llx is not 4-byte aligned",
|
|
params->device_address);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
// 安全的DMA传输函数
|
|
long calculet_dma_safe_transfer(struct calculet_dma_controller *controller,
|
|
struct calculet_dma_transfer_channels_params *params,
|
|
struct calculet_resource *resource)
|
|
{
|
|
struct calculet_dma_channel *channel;
|
|
struct calculet_dma_transfer_context *ctx;
|
|
long ret;
|
|
int ch_id;
|
|
|
|
if (!controller || !params || !resource) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
// 验证传输参数
|
|
ret = calculet_dma_validate_transfer_params(params);
|
|
if (ret) {
|
|
return ret;
|
|
}
|
|
|
|
// 检查控制器状态
|
|
if (!controller->controller_enabled) {
|
|
return -ENODEV;
|
|
}
|
|
|
|
ch_id = params->dma_channel_id;
|
|
|
|
// 获取通道
|
|
if (params->dma_transfer_direction == TRANSFER_H2C_DIRECTION) {
|
|
if (ch_id >= CALCULET_READ_CHANNELS) {
|
|
return -EINVAL;
|
|
}
|
|
channel = &controller->dma_rdch_s[ch_id];
|
|
} else {
|
|
if (ch_id >= CALCULET_WRITE_CHANNELS) {
|
|
return -EINVAL;
|
|
}
|
|
channel = &controller->dma_wrch_s[ch_id];
|
|
}
|
|
|
|
// 检查通道状态
|
|
if (!calculet_dma_channel_is_available(channel)) {
|
|
return -EBUSY;
|
|
}
|
|
|
|
// 创建传输上下文
|
|
ctx = calculet_dma_create_transfer_context(current->pid);
|
|
if (!ctx) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
// 设置通道参数
|
|
channel->dma_channel_size = params->dma_channel_size;
|
|
channel->buffer = params->host_address;
|
|
channel->dst_address = params->device_address;
|
|
channel->current_transfer = ctx;
|
|
|
|
// 增加活动传输计数
|
|
atomic_inc(&controller->active_transfers);
|
|
|
|
// 执行传输
|
|
if (params->dma_transfer_direction == TRANSFER_H2C_DIRECTION) {
|
|
ret = calculet_dma_h2c_transfer(controller->dev, channel, resource);
|
|
} else {
|
|
ret = calculet_dma_c2h_transfer(controller->dev, channel, resource);
|
|
}
|
|
|
|
// 清理
|
|
channel->current_transfer = NULL;
|
|
calculet_dma_transfer_context_put(ctx);
|
|
atomic_dec(&controller->active_transfers);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static long calculet_block_dma_transfer_ioctl(struct calculet_resource *resource, struct calculet_dma_controller *controller, unsigned long arg)
|
|
{
|
|
struct calculet_dma_transfer_channels_params input;
|
|
long ret = -1;
|
|
|
|
if (copy_from_user(&input, (void *)arg, sizeof(input))) {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, controller->dev, "copy_from_user fail");
|
|
return -EFAULT;
|
|
}
|
|
|
|
// 使用安全的传输函数
|
|
ret = calculet_dma_safe_transfer(controller, &input, resource);
|
|
|
|
return ret;
|
|
}
|
|
|
|
int calculet_dma_poll_ioctl(struct calculet_dma_controller *controller)
|
|
{
|
|
struct calculet_dma_poll *poll;
|
|
int region_data = 0;
|
|
int i;
|
|
|
|
if (!controller || !controller->dma_kthread || !controller->dma_kthread->dma_poll) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
poll = controller->dma_kthread->dma_poll;
|
|
|
|
spin_lock(&poll->poll_lock);
|
|
region_data = *poll->status_region;
|
|
|
|
if(region_data != 0) {
|
|
for(i = 0; i < CALCULET_TOTAL_CHANNELS; i++)
|
|
{
|
|
if(poll->channel_data[i] == region_data) {
|
|
if(i < CALCULET_TOTAL_CHANNELS/2) {
|
|
atomic_set(&controller->dma_rdch_s[i].transfer_finish, 1);
|
|
wake_up_interruptible(&controller->dma_rdch_s[i].channel_wq);
|
|
break;
|
|
}
|
|
else {
|
|
atomic_set(&controller->dma_wrch_s[i - CALCULET_READ_CHANNELS].transfer_finish, 1);
|
|
wake_up_interruptible(&controller->dma_wrch_s[i - CALCULET_READ_CHANNELS].channel_wq);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
*poll->status_region = 0;
|
|
}
|
|
spin_unlock(&poll->poll_lock);
|
|
|
|
return region_data;
|
|
}
|
|
|
|
long calculet_dma_ioctl(struct calculet_resource *resource, struct calculet_dma_controller *controller, unsigned int cmd, unsigned long arg)
|
|
{
|
|
if (!resource || !controller) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
switch(cmd) {
|
|
case CALCULET_BLOCK_DMA_TRANSFER:
|
|
return calculet_block_dma_transfer_ioctl(resource, controller, arg);
|
|
case CALCULET_INTERRUPT_POLL:
|
|
return calculet_dma_poll_ioctl(controller);
|
|
default:
|
|
calculet_dev_log(CALCULET_LOG_ERROR, controller->dev, "Invalid DMA ioctl code 0x%x (nr: %d)", cmd, _IOC_NR(cmd));
|
|
return -ENOTTY;
|
|
}
|
|
}
|
|
|
|
//block dma api
|
|
long calculet_dma_h2c_transfer(struct device *dev, struct calculet_dma_channel *channel, struct calculet_resource *resource)
|
|
{
|
|
int ch = channel->channel_index;
|
|
unsigned long timeout = msecs_to_jiffies(100000); // 设置超时时间为100秒
|
|
struct calculet_pcie_board *board = dev_get_drvdata(dev);
|
|
long ret = 0;
|
|
int transfer_result;
|
|
|
|
if (!board || !calculet_board_is_operational(board)) {
|
|
return -ENODEV;
|
|
}
|
|
|
|
CALCULET_PERF_START(h2c_transfer);
|
|
|
|
// 获取传输互斥锁
|
|
if (mutex_lock_interruptible(&channel->transfer_mutex)) {
|
|
return -ERESTARTSYS;
|
|
}
|
|
|
|
// 缓冲区管理
|
|
mutex_lock(&channel->buffer_mutex);
|
|
if(channel->dma_channel_size <= DMA_CHANNEL_MAX_SIZE) {
|
|
if(channel->kernel_buf == NULL) {
|
|
channel->kernel_buf = dma_alloc_coherent(dev, DMA_CHANNEL_MAX_SIZE, &channel->dma_handle, GFP_KERNEL);
|
|
if (!channel->kernel_buf) {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to allocate DMA buffer");
|
|
ret = -ENOMEM;
|
|
goto error_unlock_buffer;
|
|
}
|
|
}
|
|
} else {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "H2C transfer size %ld is too large", channel->dma_channel_size);
|
|
ret = -EINVAL;
|
|
goto error_unlock_buffer;
|
|
}
|
|
// if(channel->dma_len != channel->dma_channel_size) {
|
|
// if(channel->kernel_buf == NULL) {
|
|
// channel->kernel_buf = dma_alloc_coherent(dev, channel->dma_channel_size, &channel->dma_handle, GFP_KERNEL);
|
|
// if (!channel->kernel_buf) {
|
|
// calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to allocate DMA buffer");
|
|
// ret = -ENOMEM;
|
|
// goto error_unlock_buffer;
|
|
// }
|
|
// } else {
|
|
// dma_free_coherent(dev, channel->dma_len, channel->kernel_buf, channel->dma_handle);
|
|
// channel->kernel_buf = dma_alloc_coherent(dev, channel->dma_channel_size, &channel->dma_handle, GFP_KERNEL);
|
|
// if (!channel->kernel_buf) {
|
|
// calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to allocate DMA buffer");
|
|
// ret = -ENOMEM;
|
|
// goto error_unlock_buffer;
|
|
// }
|
|
// }
|
|
// channel->dma_len = channel->dma_channel_size;
|
|
// }
|
|
atomic_inc(&channel->buffer_users);
|
|
mutex_unlock(&channel->buffer_mutex);
|
|
|
|
// 从用户空间拷贝数据到DMA一致性内存
|
|
if (copy_from_user(channel->kernel_buf, (void __user *)channel->buffer, channel->dma_channel_size)) {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to copy_from_user");
|
|
ret = -EFAULT;
|
|
goto error_cleanup;
|
|
}
|
|
|
|
// 配置DMA寄存器
|
|
iowrite32(0x10, (u8*)resource->address + DMA_RDCH_CONTROL1_OFF(ch));
|
|
iowrite32(PCI_DMA_L(channel->dma_channel_size), (u8*)resource->address + DMA_RDCH_SIZE_OFF(ch));
|
|
iowrite32(PCI_DMA_L(channel->dma_handle), (u8*)resource->address + DMA_RDCH_SAR_LOW_OFF(ch));
|
|
iowrite32(PCI_DMA_H(channel->dma_handle), (u8*)resource->address + DMA_RDCH_SAR_HIGH_OFF(ch));
|
|
iowrite32(PCI_DMA_L(channel->dst_address), (u8*)resource->address + DMA_RDCH_DAR_LOW_OFF(ch));
|
|
iowrite32(PCI_DMA_H(channel->dst_address), (u8*)resource->address + DMA_RDCH_DAR_HIGH_OFF(ch));
|
|
|
|
// 重置完成标志
|
|
atomic_set(&channel->transfer_finish, 0);
|
|
|
|
// 使用资源结构体中的锁
|
|
mutex_lock(&resource->dma_global_lock);
|
|
iowrite32(ch, (u8*)resource->address + DMA_READ_DOORBELL_OFF);
|
|
mutex_unlock(&resource->dma_global_lock);
|
|
|
|
// 等待传输完成或超时
|
|
transfer_result = wait_event_interruptible_timeout(channel->channel_wq,
|
|
atomic_read(&channel->transfer_finish) != 0,
|
|
timeout);
|
|
if (transfer_result > 0) {
|
|
int finish_status = atomic_read(&channel->transfer_finish);
|
|
if (finish_status == 1) {
|
|
// 传输成功
|
|
calculet_board_stats_update_dma_h2c(board, channel->dma_channel_size);
|
|
calculet_dma_update_channel_stats(channel, channel->dma_channel_size, true);
|
|
ret = 0;
|
|
} else {
|
|
// 传输错误
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "H2C transfer failed with status %d", finish_status);
|
|
calculet_board_stats_update_error(board);
|
|
calculet_dma_update_channel_stats(channel, 0, false);
|
|
ret = finish_status;
|
|
}
|
|
} else if (transfer_result == 0) {
|
|
// 超时
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "H2C transfer timed out");
|
|
calculet_board_stats_update_error(board);
|
|
calculet_dma_update_channel_stats(channel, 0, false);
|
|
ret = -ETIMEDOUT;
|
|
} else {
|
|
// 被信号中断
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "H2C transfer interrupted");
|
|
ret = -ERESTARTSYS;
|
|
}
|
|
|
|
CALCULET_PERF_END(h2c_transfer, "channel %d, size %ld, result %ld",
|
|
ch, channel->dma_channel_size, ret);
|
|
|
|
calculet_dev_log(CALCULET_LOG_DEBUG, dev, "H2C transfer: channel %d, src 0x%llx, dst 0x%llx, size %ld, result %ld",
|
|
ch, channel->dma_handle, channel->dst_address, channel->dma_channel_size, ret);
|
|
|
|
error_cleanup:
|
|
mutex_lock(&channel->buffer_mutex);
|
|
if (atomic_dec_and_test(&channel->buffer_users)) {
|
|
// 如果发生错误，释放缓冲区
|
|
if (ret != 0 && channel->kernel_buf) {
|
|
dma_free_coherent(dev, channel->dma_channel_size, channel->kernel_buf, channel->dma_handle);
|
|
channel->kernel_buf = NULL;
|
|
channel->dma_len = 0;
|
|
}
|
|
}
|
|
|
|
error_unlock_buffer:
|
|
mutex_unlock(&channel->buffer_mutex);
|
|
mutex_unlock(&channel->transfer_mutex);
|
|
|
|
return ret;
|
|
}
|
|
|
|
long calculet_dma_c2h_transfer(struct device *dev, struct calculet_dma_channel *channel, struct calculet_resource *resource)
|
|
{
|
|
int ch = channel->channel_index;
|
|
unsigned long timeout = msecs_to_jiffies(100000); // 设置超时时间为100秒
|
|
struct calculet_pcie_board *board = dev_get_drvdata(dev);
|
|
long ret = 0;
|
|
int transfer_result;
|
|
|
|
if (!board || !calculet_board_is_operational(board)) {
|
|
return -ENODEV;
|
|
}
|
|
|
|
CALCULET_PERF_START(c2h_transfer);
|
|
|
|
// 获取传输互斥锁
|
|
if (mutex_lock_interruptible(&channel->transfer_mutex)) {
|
|
return -ERESTARTSYS;
|
|
}
|
|
|
|
// 缓冲区管理
|
|
mutex_lock(&channel->buffer_mutex);
|
|
if(channel->dma_channel_size <= DMA_CHANNEL_MAX_SIZE) {
|
|
if(channel->kernel_buf == NULL) {
|
|
channel->kernel_buf = dma_alloc_coherent(dev, DMA_CHANNEL_MAX_SIZE, &channel->dma_handle, GFP_KERNEL);
|
|
if (!channel->kernel_buf) {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to allocate DMA buffer");
|
|
ret = -ENOMEM;
|
|
goto error_unlock_buffer;
|
|
}
|
|
}
|
|
} else {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "C2H transfer size %ld is too large", channel->dma_channel_size);
|
|
ret = -EINVAL;
|
|
goto error_unlock_buffer;
|
|
}
|
|
// if(channel->dma_len != channel->dma_channel_size) {
|
|
// if(channel->kernel_buf == NULL) {
|
|
// channel->kernel_buf = dma_alloc_coherent(dev, channel->dma_channel_size, &channel->dma_handle, GFP_KERNEL);
|
|
// if (!channel->kernel_buf) {
|
|
// calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to allocate DMA buffer");
|
|
// ret = -ENOMEM;
|
|
// goto error_unlock_buffer;
|
|
// }
|
|
// } else {
|
|
// dma_free_coherent(dev, channel->dma_len, channel->kernel_buf, channel->dma_handle);
|
|
// channel->kernel_buf = dma_alloc_coherent(dev, channel->dma_channel_size, &channel->dma_handle, GFP_KERNEL);
|
|
// if (!channel->kernel_buf) {
|
|
// calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to allocate DMA buffer");
|
|
// ret = -ENOMEM;
|
|
// goto error_unlock_buffer;
|
|
// }
|
|
// }
|
|
// channel->dma_len = channel->dma_channel_size;
|
|
// }
|
|
atomic_inc(&channel->buffer_users);
|
|
mutex_unlock(&channel->buffer_mutex);
|
|
|
|
// 配置DMA寄存器
|
|
iowrite32(0x10, (u8*)resource->address + DMA_WRCH_CONTROL1_OFF(ch));
|
|
iowrite32(PCI_DMA_L(channel->dma_channel_size), (u8*)resource->address + DMA_WRCH_SIZE_OFF(ch));
|
|
iowrite32(PCI_DMA_L(channel->dst_address), (u8*)resource->address + DMA_WRCH_SAR_LOW_OFF(ch));
|
|
iowrite32(PCI_DMA_H(channel->dst_address), (u8*)resource->address + DMA_WRCH_SAR_HIGH_OFF(ch));
|
|
iowrite32(PCI_DMA_L(channel->dma_handle), (u8*)resource->address + DMA_WRCH_DAR_LOW_OFF(ch));
|
|
iowrite32(PCI_DMA_H(channel->dma_handle), (u8*)resource->address + DMA_WRCH_DAR_HIGH_OFF(ch));
|
|
|
|
// 重置完成标志
|
|
atomic_set(&channel->transfer_finish, 0);
|
|
|
|
// 使用资源结构体中的锁
|
|
mutex_lock(&resource->dma_global_lock);
|
|
iowrite32(ch, (u8*)resource->address + DMA_WRITE_DOORBELL_OFF);
|
|
mutex_unlock(&resource->dma_global_lock);
|
|
|
|
// 等待传输完成或超时
|
|
transfer_result = wait_event_interruptible_timeout(channel->channel_wq,
|
|
atomic_read(&channel->transfer_finish) != 0,
|
|
timeout);
|
|
if (transfer_result > 0) {
|
|
int finish_status = atomic_read(&channel->transfer_finish);
|
|
if (finish_status == 1) {
|
|
// 传输成功，将数据从DMA一致性内存复制回用户空间
|
|
if (copy_to_user((void __user *)channel->buffer, channel->kernel_buf, channel->dma_channel_size)) {
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "Failed to copy_to_user");
|
|
calculet_board_stats_update_error(board);
|
|
calculet_dma_update_channel_stats(channel, 0, false);
|
|
ret = -EFAULT;
|
|
} else {
|
|
calculet_board_stats_update_dma_c2h(board, channel->dma_channel_size);
|
|
calculet_dma_update_channel_stats(channel, channel->dma_channel_size, true);
|
|
ret = 0;
|
|
}
|
|
} else {
|
|
// 传输错误
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "C2H transfer failed with status %d", finish_status);
|
|
calculet_board_stats_update_error(board);
|
|
calculet_dma_update_channel_stats(channel, 0, false);
|
|
ret = finish_status;
|
|
}
|
|
} else if (transfer_result == 0) {
|
|
// 超时
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "C2H transfer timed out");
|
|
calculet_board_stats_update_error(board);
|
|
calculet_dma_update_channel_stats(channel, 0, false);
|
|
ret = -ETIMEDOUT;
|
|
} else {
|
|
// 被信号中断
|
|
calculet_dev_log(CALCULET_LOG_ERROR, dev, "C2H transfer interrupted");
|
|
ret = -ERESTARTSYS;
|
|
}
|
|
|
|
CALCULET_PERF_END(c2h_transfer, "channel %d, size %ld, result %ld",
|
|
ch, channel->dma_channel_size, ret);
|
|
|
|
calculet_dev_log(CALCULET_LOG_DEBUG, dev, "C2H transfer: channel %d, src 0x%llx, dst 0x%llx, size %ld, result %ld",
|
|
ch, channel->dst_address, channel->dma_handle, channel->dma_channel_size, ret);
|
|
|
|
mutex_lock(&channel->buffer_mutex);
|
|
if (atomic_dec_and_test(&channel->buffer_users)) {
|
|
// 如果发生错误，释放缓冲区
|
|
if (ret != 0 && channel->kernel_buf) {
|
|
dma_free_coherent(dev, channel->dma_channel_size, channel->kernel_buf, channel->dma_handle);
|
|
channel->kernel_buf = NULL;
|
|
channel->dma_len = 0;
|
|
}
|
|
}
|
|
|
|
error_unlock_buffer:
|
|
mutex_unlock(&channel->buffer_mutex);
|
|
mutex_unlock(&channel->transfer_mutex);
|
|
|
|
return ret;
|
|
}
|
|
|
|
irqreturn_t calculet_irqhandler(int irq, void *dev_id)
|
|
{
|
|
irqreturn_t return_value = IRQ_HANDLED;
|
|
struct calculet_dma_channel *channel = (struct calculet_dma_channel *)dev_id;
|
|
struct calculet_pcie_board *board;
|
|
|
|
if (!channel) {
|
|
return IRQ_NONE;
|
|
}
|
|
|
|
calculet_log(CALCULET_LOG_DEBUG, "IRQ %d for channel %d (vector %d)",
|
|
irq, channel->channel_index, channel->vector_index);
|
|
|
|
// 设置传输完成标志
|
|
atomic_set(&channel->transfer_finish, 1);
|
|
|
|
// 唤醒等待的线程
|
|
wake_up_interruptible(&channel->channel_wq);
|
|
|
|
// 更新中断统计（简化版本）
|
|
board = container_of(channel, struct calculet_pcie_board, cdma.dma_rdch_s[channel->channel_index]);
|
|
if (!board) {
|
|
board = container_of(channel, struct calculet_pcie_board, cdma.dma_wrch_s[channel->channel_index]);
|
|
}
|
|
if (board) {
|
|
calculet_board_stats_update_interrupt(board);
|
|
}
|
|
|
|
return return_value;
|
|
}
|
|
|
|
int calculet_poll_handler(void *data)
|
|
{
|
|
struct calculet_dma_controller *controller = (struct calculet_dma_controller *)data;
|
|
struct calculet_dma_poll *poll = controller->dma_kthread->dma_poll;
|
|
int region_data = 0;
|
|
int i;
|
|
|
|
atomic_set(&poll->poll_active, 1);
|
|
|
|
while(!kthread_should_stop() && !atomic_read(&controller->dma_kthread->should_stop))
|
|
{
|
|
spin_lock(&poll->poll_lock);
|
|
region_data = *poll->status_region;
|
|
|
|
if(region_data != 0) {
|
|
for(i = 0; i < CALCULET_TOTAL_CHANNELS; i++)
|
|
{
|
|
if(poll->channel_data[i] == region_data) {
|
|
if(i < CALCULET_TOTAL_CHANNELS/2) {
|
|
atomic_set(&controller->dma_rdch_s[i].transfer_finish, 1);
|
|
wake_up_interruptible(&controller->dma_rdch_s[i].channel_wq);
|
|
break;
|
|
}
|
|
else {
|
|
atomic_set(&controller->dma_wrch_s[i - CALCULET_READ_CHANNELS].transfer_finish, 1);
|
|
wake_up_interruptible(&controller->dma_wrch_s[i - CALCULET_READ_CHANNELS].channel_wq);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
*poll->status_region = 0;
|
|
}
|
|
spin_unlock(&poll->poll_lock);
|
|
|
|
schedule();
|
|
}
|
|
|
|
atomic_set(&poll->poll_active, 0);
|
|
complete(&controller->dma_kthread->thread_completion);
|
|
|
|
calculet_log(CALCULET_LOG_DEBUG, "Poll handler thread exiting");
|
|
return 0;
|
|
}
|
|
|
|
// 统计信息更新（简化版本）
|
|
void calculet_dma_update_channel_stats(struct calculet_dma_channel *channel,
|
|
size_t bytes, bool success)
|
|
{
|
|
if (!channel) {
|
|
return;
|
|
}
|
|
|
|
atomic64_inc(&channel->transfer_count);
|
|
if (success) {
|
|
atomic64_add(bytes, &channel->transfer_bytes);
|
|
}
|
|
|
|
channel->last_used = ktime_get();
|
|
}
|
|
|
|
// 控制器状态转储
|
|
void calculet_dma_dump_controller_state(struct calculet_dma_controller *controller)
|
|
{
|
|
int i;
|
|
|
|
if (!controller) {
|
|
return;
|
|
}
|
|
|
|
calculet_log(CALCULET_LOG_DEBUG, "DMA Controller State Dump");
|
|
calculet_log(CALCULET_LOG_DEBUG, " Enabled: %s", controller->controller_enabled ? "YES" : "NO");
|
|
calculet_log(CALCULET_LOG_DEBUG, " Active Transfers: %d", atomic_read(&controller->active_transfers));
|
|
calculet_log(CALCULET_LOG_DEBUG, " Read Channels Bitmap: 0x%lx", controller->read_channels_bitmap);
|
|
calculet_log(CALCULET_LOG_DEBUG, " Write Channels Bitmap: 0x%lx", controller->write_channels_bitmap);
|
|
|
|
// 转储读通道状态
|
|
calculet_log(CALCULET_LOG_DEBUG, " Read Channels:");
|
|
for (i = 0; i < CALCULET_READ_CHANNELS; i++) {
|
|
struct calculet_dma_channel *ch = &controller->dma_rdch_s[i];
|
|
calculet_log(CALCULET_LOG_DEBUG, " CH%d: state=%d, transfers=%lld, bytes=%lld",
|
|
i, calculet_dma_channel_get_state(ch),
|
|
atomic64_read(&ch->transfer_count),
|
|
atomic64_read(&ch->transfer_bytes));
|
|
}
|
|
|
|
// 转储写通道状态
|
|
calculet_log(CALCULET_LOG_DEBUG, " Write Channels:");
|
|
for (i = 0; i < CALCULET_WRITE_CHANNELS; i++) {
|
|
struct calculet_dma_channel *ch = &controller->dma_wrch_s[i];
|
|
calculet_log(CALCULET_LOG_DEBUG, " CH%d: state=%d, transfers=%lld, bytes=%lld",
|
|
i, calculet_dma_channel_get_state(ch),
|
|
atomic64_read(&ch->transfer_count),
|
|
atomic64_read(&ch->transfer_bytes));
|
|
}
|
|
} |