保留Linux内存的初始化原理及应用实战

1. 概述

在linux启动过程中会打印出如下信息，这些信息为我们呈现出系统下的保留内存空间情况。

Reserved memory: created DMA memory pool at 0x4c000000, size 8 MiB
OF: reserved mem: initialized node vram@4c000000, compatible id shared-dma-pool

本文只介绍基本的保留内存，不涉及CMA部分内容

保留内存的初始化流程如下图所示：

本文所说的保留内存的作用，概况来讲包括如下四方面：

1. 驱动程序特定使用
2. 加载固件到指定内存
3. DDR中某段内存区域存放特定数据，如多核处理相关代码
4. 调试驱动

2. 保留内存初始化流程

从setup_arch()函数中我们可以发现，保留内存初始化是在设备树释放之前，通过解析FDT，获取保留内存的参数来进行初始化。

void __init setup_arch(char **cmdline_p)
{
...
 arm_memblock_init(mdesc);
...
 unflatten_device_tree();
...

2.1 解析内核中的保留内存空间

在各平台初始化过程中调用early_init_fdt_scan_reserved_mem()进行保留内存的初始化。

setup_arch_memory in init.c (arch\\arc\\mm) :  early_init_fdt_scan_reserved_mem();
arm64_memblock_init in init.c (arch\\arm64\\mm) :  early_init_fdt_scan_reserved_mem();
arm_memblock_init in init.c (arch\\arm\\mm) :  early_init_fdt_scan_reserved_mem();
setup_bootmem in init.c (arch\\riscv\\mm) :  early_init_fdt_scan_reserved_mem();
bootmem_init in init.c (arch\\xtensa\\mm) :  early_init_fdt_scan_reserved_mem();
sh_of_mem_reserve in of-generic.c (arch\\sh\\boards) :  early_init_fdt_scan_reserved_mem();
of_fdt.h (include\\linux) line 66 : extern void early_init_fdt_scan_reserved_mem(void);
of_fdt.h (include\\linux) line 94 : static inline void early_init_fdt_scan_reserved_mem(void) {}
early_reserve_mem_dt in prom.c (arch\\powerpc\\kernel) :  early_init_fdt_scan_reserved_mem();
csky_memblock_init in setup.c (arch\\csky\\kernel) :  early_init_fdt_scan_reserved_mem();
bootmem_init in setup.c (arch\\h8300\\kernel) :  early_init_fdt_scan_reserved_mem();
arch_mem_init in setup.c (arch\\mips\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_memory in setup.c (arch\\nds32\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_arch in setup.c (arch\\nios2\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_memory in setup.c (arch\\openrisc\\kernel) :  early_init_fdt_scan_reserved_mem();

它的定义位于drivers/of/fdt.c中，需要内核配置打开CONFIG_OF_EARLY_FLATTREE宏。

主体函数如下：

void __init early_init_fdt_scan_reserved_mem(void)
{
 int n;
 u64 base, size;

 if (!initial_boot_params)
  return;

 for (n = 0; ; n++) {
  fdt_get_mem_rsv(initial_boot_params, n, &base, &size);
  if (!size)
   break;
  early_init_dt_reserve_memory_arch(base, size, false);
 }

 of_scan_flat_dt(__fdt_scan_reserved_mem, NULL);
 fdt_init_reserved_mem();
}

2.1.1 解析memreserve

从early_init_fdt_scan_reserved_mem()函数中的initial_boot_params可以再次确定这一点。initial_boot_params代表的是fdt的地址，如下：

## Flattened Device Tree blob at 41000000
   Booting using the fdt blob at 0x41000000
   Loading Kernel Image ... OK
   Loading Device Tree to 4ffef000, end 4ffffff2 ... OK

Starting kernel ...

Uncompressing Linux... done, booting the kernel.
[    0.000000] Booting Linux on physical CPU 0xa00
...
[    0.000000] --- initial_boot_params(fdt addr 0x4ffef000)

通过fdt_get_mem_rsv()解析设备树中的/memreserve/fields，例如树莓派处理器的设备树中定义了该属性，通常来讲，这部分内存区域是存放和rom或者多核启动相关的程序，需要注意的是内核无法使用这部分内存。这是和reserver memory的区别。

/memreserve/ 0x00000000 0x00001000;
...
/ {
        compatible = "brcm,bcm2835";
...

若在设备树中查找到了memreserve且未进行映射，则通过memblock_reserve()将这部分内存区域加入到memblock.reserved，当进行memblock到buddy转换时，释放掉memblock.reserved所标记的内存区域。

int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
{
 phys_addr_t end = base + size - 1;

 memblock_dbg("memblock_reserve: [%pa-%pa] %pS\\n",
       &base, &end, (void *)_RET_IP_);

 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
}

2.1.2 解析reserve memory

通过__fdt_scan_reserved_mem()解析设备树中保留内存相关的结点信息。

static int __init __fdt_scan_reserved_mem(unsigned long node, const char *uname,
       int depth, void *data)
{
 static int found;
 int err;

 if (!found && depth == 1 && strcmp(uname, "reserved-memory") == 0) {
  ...
 }

 if (!of_fdt_device_is_available(initial_boot_params, node))
  return 0;

 err = __reserved_mem_reserve_reg(node, uname);
 if (err == -ENOENT && of_get_flat_dt_prop(node, "size", NULL))
  fdt_reserved_mem_save_node(node, uname, 0, 0);
...
}

该函数首先解析设备树中reserved-memory结点并确认是否有效。若有效，继续检查reg或size属性定义的内存区域，通过fdt_reserved_mem_save_node()将内存信息更新到数据结构struct reserved_mem。

void __init fdt_reserved_mem_save_node(unsigned long node, const char *uname,phys_addr_t base, phys_addr_t size)
{
 struct reserved_mem *rmem = &reserved_mem[reserved_mem_count];

 if (reserved_mem_count == ARRAY_SIZE(reserved_mem)) {
  pr_err("not enough space all defined regions.\\n");
  return;
 }

 rmem- >fdt_node = node;
 rmem- >name = uname;
 rmem- >base = base;
 rmem- >size = size;

 reserved_mem_count++;
 return;
}

2.2 保留内存初始化

保留内存初始化的主体函数是fdt_init_reserved_mem()，其首先解析设备树结点no-map、phandle等信息，最后通过关键函数__reserved_mem_init_node()完成保留内存子节点的初始化。

放开解析保留内存解析相关的打印，再回头再看kernel的启动信息，启动信息中保留内存相关内容正是此处打印出来的。

OF: fdt: Reserved memory: reserved region for node 'vram@4c000000': base 0x4c000000, size 8 MiB
Reserved memory: created DMA memory pool at 0x4c000000, size 8 MiB

created DMA memory pool...来自函数rmem_dma_setup()，这个函数从数据结构struct reserved_mem获取保留内存的信息。

static int __init rmem_dma_setup(struct reserved_mem *rmem)
{
 unsigned long node = rmem- >fdt_node;

 if (of_get_flat_dt_prop(node, "reusable", NULL))
  return -EINVAL;

#ifdef CONFIG_ARM
 if (!of_get_flat_dt_prop(node, "no-map", NULL)) {
  pr_err("Reserved memory: regions without no-map are not yet supported\\n");
  return -EINVAL;
 }

 if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {
  WARN(dma_reserved_default_memory,
       "Reserved memory: region for default DMA coherent area is redefined\\n");
  dma_reserved_default_memory = rmem;
 }
#endif

 rmem- >ops = &rmem_dma_ops;
 pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\\n",
  &rmem- >base, (unsigned long)rmem- >size / SZ_1M);
 return 0;
}

在函数rmem_dma_setup()中还会例化reserved_mem.ops，如下：

static const struct reserved_mem_ops rmem_dma_ops = {
 .device_init = rmem_dma_device_init,
 .device_release = rmem_dma_device_release,
};

3. 设备树中保留内存的定义方式

以vexpress-v2p-ca9.dts中保留内存的定义方式为例，说明dts文件中如何定义保留内存。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        /* Chipselect 3 is physically at 0x4c000000 */
        vram: vram@4c000000 {
                /* 8 MB of designated video RAM */
                compatible = "shared-dma-pool";
                reg = < 0x4c000000 0x00800000 >;
                no-map;
        };
};

保留内存由根节点和1个或多个子结点组成。

根节点包括如下信息：

• #address-cells、#size-cells

  必须项，需要同dts根节点中相关属性保持一致。

/dts-v1/;
#include "vexpress-v2m.dtsi"

/ {
        model = "V2P-CA9";
        arm,hbi = < 0x191 >;
        arm,vexpress,site = < 0xf >;
        compatible = "arm,vexpress,v2p-ca9", "arm,vexpress";
        interrupt-parent = < &gic >;
        #address-cells = < 1 >;
        #size-cells = < 1 >;
...

• ranges

  必须项，且定义为空

子结点包括如下信息：

• 空间大小

  可以通过reg或size来指定保留内存空间大小，若二者同时存在，以reg属性为准。通过size的方式如下：

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        mfc_left: region_mfc_left {
                compatible = "shared-dma-pool";
                no-map;
                size = < 0x2400000 >;
...

• alignment

  可选项

• alloc-ranges

  可选项，通常可以和size同时使用。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;
        default-pool {
                compatible = "shared-dma-pool";
                size = < 0x6000000 >;
                alloc-ranges = < 0x40000000 0x10000000 >;
...

• compatible

  可能包括shared-dma-pool或者shared-dma-pool。

  主要关注shared-dma-pool，当驱动程序需要申请DMA空间时，可以从这里进行申请内存空间。

• no-map

  该属性意为不会为这段内存创建地址映射，在使用之前，需要调用者通过ioremap创建页表映射关系才可以正常访问。这个属性与reusable是互斥的。

• no-map-fixup

  保持内存映射。

• reusable

  当驱动程序不使用这些内存的时候，OS可以使用这些内存。

• linux,cma-default

  定义该段保留内存空间是默认的CMA内存池。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;
        default-pool {
                compatible = "shared-dma-pool";
                size = < 0x6000000 >;
                alloc-ranges = < 0x40000000 0x10000000 >;
                reusable;
                linux,cma-default;
        };
};

4. 保留内存的使用

4.1 设备树编码

定义保留内存：

memory@60000000 {
        device_type = "memory";
        reg = < 0x60000000 0x40000000 >;
};

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        /* test reserve memory */
        test_reserve: test_reserve@90000000 {
                /* 1 MB reserve memory */
                compatible = "shared-dma-pool";
                reg = < 0x90000000 0x00100000 >;
                no-map;
        };
};

定义memory-region，将保留内存指定给特定设备，如下：

driver-test@8000 {
        /* compatible = "test_driver_0", "simple_bus"; */
        compatible = "test_driver_0";
        reg = < 0x80008000 0x1000 >;

        interrupt-parent = < &gic >;
        interrupts= < 0 89 4 >, < 0 90 4444 >;
        interrupt-names = "first_irq", "second_irq";

        clocks = < &oscclk2 >;
        clock-names = "apb_pclk";

        memory-region = < &test_reserve >;

        status = "okay";

        simple_bus_test{
            compatile = "simple_bus_test";
        };
};

加载kernel后，保留内存相关的打印信息如下：

Reserved memory: created DMA memory pool at 0x90000000, size 1 MiB
OF: reserved mem: initialized node test_reserve@90000000, compatible id shared-dma-pool

4.2 驱动程序编码

驱动程序中调用of_reserved_mem_device_init()申请保留内存空间。

static inline int of_reserved_mem_device_init(struct device *dev)
{
 return of_reserved_mem_device_init_by_idx(dev, dev- >of_node, 0);
}

主体函数是of_reserved_mem_device_init_by_idx()

int of_reserved_mem_device_init_by_idx(struct device *dev,
           struct device_node *np, int idx)
{
 struct rmem_assigned_device *rd;
 struct device_node *target;
 struct reserved_mem *rmem;
 int ret;

 if (!np || !dev)
  return -EINVAL;

 target = of_parse_phandle(np, "memory-region", idx);
 if (!target)
  return -ENODEV;

 if (!of_device_is_available(target)) {
  of_node_put(target);
  return 0;
 }

 rmem = __find_rmem(target);
 of_node_put(target);

 if (!rmem || !rmem- >ops || !rmem- >ops- >device_init)
  return -EINVAL;

 rd = kmalloc(sizeof(struct rmem_assigned_device), GFP_KERNEL);
 if (!rd)
  return -ENOMEM;

 ret = rmem- >ops- >device_init(rmem, dev);
 if (ret == 0) {
  rd- >dev = dev;
  rd- >rmem = rmem;

  mutex_lock(&of_rmem_assigned_device_mutex);
  list_add(&rd- >list, &of_rmem_assigned_device_list);
  mutex_unlock(&of_rmem_assigned_device_mutex);

  dev_info(dev, "assigned reserved memory node %s\\n", rmem- >name);
 } else {
  kfree(rd);
 }

 return ret;
}

然后可以通过dma_alloc_coherent()在保留内存空间申请DMA空间。

void *dma_vaddr;
dma_addr_t dma_handler;

/* Start: test reserve memory */
ret = of_reserved_mem_device_init(&pdev- >dev);
if (!ret) {
        dev_info(&pdev- >dev, "using device-specific reserved memory\\n");
}

dma_set_coherent_mask(&pdev- >dev, 0xFFFFFFFF);
dma_vaddr = dma_alloc_coherent(&pdev- >dev, 64*1024, &dma_handler, GFP_KERNEL);
if (!dma_vaddr) {
        pr_notice("DMA allocation failed\\n");
        return false;
}
dev_info(&pdev- >dev, "DMA alloc phy addr 0x%X\\n", (u32)dma_handler);
/* End: test reserve memory */

执行结果：

test_driver ahb:driver-test@8000: assigned reserved memory node test_reserve@90000000
test_driver ahb:driver-test@8000: using device-specific reserved memory
test_driver ahb:driver-test@8000: DMA alloc phy addr 0x90000000

结果表明，DMA申请的内存落于保留内存空间0x90000000-0x90100000。