455 lines
12 KiB
C
455 lines
12 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* iommu.c: IOMMU specific routines for memory management.
|
|
*
|
|
* Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
|
|
* Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
|
|
* Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
|
|
* Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/init.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/dma-map-ops.h>
|
|
#include <linux/of.h>
|
|
#include <linux/of_device.h>
|
|
|
|
#include <asm/io.h>
|
|
#include <asm/mxcc.h>
|
|
#include <asm/mbus.h>
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/tlbflush.h>
|
|
#include <asm/bitext.h>
|
|
#include <asm/iommu.h>
|
|
#include <asm/dma.h>
|
|
|
|
#include "mm_32.h"
|
|
|
|
/*
|
|
* This can be sized dynamically, but we will do this
|
|
* only when we have a guidance about actual I/O pressures.
|
|
*/
|
|
#define IOMMU_RNGE IOMMU_RNGE_256MB
|
|
#define IOMMU_START 0xF0000000
|
|
#define IOMMU_WINSIZE (256*1024*1024U)
|
|
#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */
|
|
#define IOMMU_ORDER 6 /* 4096 * (1<<6) */
|
|
|
|
static int viking_flush;
|
|
/* viking.S */
|
|
extern void viking_flush_page(unsigned long page);
|
|
extern void viking_mxcc_flush_page(unsigned long page);
|
|
|
|
/*
|
|
* Values precomputed according to CPU type.
|
|
*/
|
|
static unsigned int ioperm_noc; /* Consistent mapping iopte flags */
|
|
static pgprot_t dvma_prot; /* Consistent mapping pte flags */
|
|
|
|
#define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
|
|
#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
|
|
|
|
static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
|
|
static const struct dma_map_ops sbus_iommu_dma_pflush_ops;
|
|
|
|
static void __init sbus_iommu_init(struct platform_device *op)
|
|
{
|
|
struct iommu_struct *iommu;
|
|
unsigned int impl, vers;
|
|
unsigned long *bitmap;
|
|
unsigned long control;
|
|
unsigned long base;
|
|
unsigned long tmp;
|
|
|
|
iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
|
|
if (!iommu) {
|
|
prom_printf("Unable to allocate iommu structure\n");
|
|
prom_halt();
|
|
}
|
|
|
|
iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
|
|
"iommu_regs");
|
|
if (!iommu->regs) {
|
|
prom_printf("Cannot map IOMMU registers\n");
|
|
prom_halt();
|
|
}
|
|
|
|
control = sbus_readl(&iommu->regs->control);
|
|
impl = (control & IOMMU_CTRL_IMPL) >> 28;
|
|
vers = (control & IOMMU_CTRL_VERS) >> 24;
|
|
control &= ~(IOMMU_CTRL_RNGE);
|
|
control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
|
|
sbus_writel(control, &iommu->regs->control);
|
|
|
|
iommu_invalidate(iommu->regs);
|
|
iommu->start = IOMMU_START;
|
|
iommu->end = 0xffffffff;
|
|
|
|
/* Allocate IOMMU page table */
|
|
/* Stupid alignment constraints give me a headache.
|
|
We need 256K or 512K or 1M or 2M area aligned to
|
|
its size and current gfp will fortunately give
|
|
it to us. */
|
|
tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
|
|
if (!tmp) {
|
|
prom_printf("Unable to allocate iommu table [0x%lx]\n",
|
|
IOMMU_NPTES * sizeof(iopte_t));
|
|
prom_halt();
|
|
}
|
|
iommu->page_table = (iopte_t *)tmp;
|
|
|
|
/* Initialize new table. */
|
|
memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
|
|
flush_cache_all();
|
|
flush_tlb_all();
|
|
|
|
base = __pa((unsigned long)iommu->page_table) >> 4;
|
|
sbus_writel(base, &iommu->regs->base);
|
|
iommu_invalidate(iommu->regs);
|
|
|
|
bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
|
|
if (!bitmap) {
|
|
prom_printf("Unable to allocate iommu bitmap [%d]\n",
|
|
(int)(IOMMU_NPTES>>3));
|
|
prom_halt();
|
|
}
|
|
bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
|
|
/* To be coherent on HyperSparc, the page color of DVMA
|
|
* and physical addresses must match.
|
|
*/
|
|
if (srmmu_modtype == HyperSparc)
|
|
iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
|
|
else
|
|
iommu->usemap.num_colors = 1;
|
|
|
|
printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
|
|
impl, vers, iommu->page_table,
|
|
(int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
|
|
|
|
op->dev.archdata.iommu = iommu;
|
|
|
|
if (flush_page_for_dma_global)
|
|
op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
|
|
else
|
|
op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
|
|
}
|
|
|
|
static int __init iommu_init(void)
|
|
{
|
|
struct device_node *dp;
|
|
|
|
for_each_node_by_name(dp, "iommu") {
|
|
struct platform_device *op = of_find_device_by_node(dp);
|
|
|
|
sbus_iommu_init(op);
|
|
of_propagate_archdata(op);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
subsys_initcall(iommu_init);
|
|
|
|
/* Flush the iotlb entries to ram. */
|
|
/* This could be better if we didn't have to flush whole pages. */
|
|
static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
|
|
{
|
|
unsigned long start;
|
|
unsigned long end;
|
|
|
|
start = (unsigned long)iopte;
|
|
end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
|
|
start &= PAGE_MASK;
|
|
if (viking_mxcc_present) {
|
|
while(start < end) {
|
|
viking_mxcc_flush_page(start);
|
|
start += PAGE_SIZE;
|
|
}
|
|
} else if (viking_flush) {
|
|
while(start < end) {
|
|
viking_flush_page(start);
|
|
start += PAGE_SIZE;
|
|
}
|
|
} else {
|
|
while(start < end) {
|
|
__flush_page_to_ram(start);
|
|
start += PAGE_SIZE;
|
|
}
|
|
}
|
|
}
|
|
|
|
static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
|
|
unsigned long offset, size_t len, bool per_page_flush)
|
|
{
|
|
struct iommu_struct *iommu = dev->archdata.iommu;
|
|
phys_addr_t paddr = page_to_phys(page) + offset;
|
|
unsigned long off = paddr & ~PAGE_MASK;
|
|
unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
|
|
unsigned long pfn = __phys_to_pfn(paddr);
|
|
unsigned int busa, busa0;
|
|
iopte_t *iopte, *iopte0;
|
|
int ioptex, i;
|
|
|
|
/* XXX So what is maxphys for us and how do drivers know it? */
|
|
if (!len || len > 256 * 1024)
|
|
return DMA_MAPPING_ERROR;
|
|
|
|
/*
|
|
* We expect unmapped highmem pages to be not in the cache.
|
|
* XXX Is this a good assumption?
|
|
* XXX What if someone else unmaps it here and races us?
|
|
*/
|
|
if (per_page_flush && !PageHighMem(page)) {
|
|
unsigned long vaddr, p;
|
|
|
|
vaddr = (unsigned long)page_address(page) + offset;
|
|
for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
|
|
flush_page_for_dma(p);
|
|
}
|
|
|
|
/* page color = pfn of page */
|
|
ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
|
|
if (ioptex < 0)
|
|
panic("iommu out");
|
|
busa0 = iommu->start + (ioptex << PAGE_SHIFT);
|
|
iopte0 = &iommu->page_table[ioptex];
|
|
|
|
busa = busa0;
|
|
iopte = iopte0;
|
|
for (i = 0; i < npages; i++) {
|
|
iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
|
|
iommu_invalidate_page(iommu->regs, busa);
|
|
busa += PAGE_SIZE;
|
|
iopte++;
|
|
pfn++;
|
|
}
|
|
|
|
iommu_flush_iotlb(iopte0, npages);
|
|
return busa0 + off;
|
|
}
|
|
|
|
static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
|
|
struct page *page, unsigned long offset, size_t len,
|
|
enum dma_data_direction dir, unsigned long attrs)
|
|
{
|
|
flush_page_for_dma(0);
|
|
return __sbus_iommu_map_page(dev, page, offset, len, false);
|
|
}
|
|
|
|
static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
|
|
struct page *page, unsigned long offset, size_t len,
|
|
enum dma_data_direction dir, unsigned long attrs)
|
|
{
|
|
return __sbus_iommu_map_page(dev, page, offset, len, true);
|
|
}
|
|
|
|
static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
|
|
int nents, enum dma_data_direction dir, unsigned long attrs,
|
|
bool per_page_flush)
|
|
{
|
|
struct scatterlist *sg;
|
|
int j;
|
|
|
|
for_each_sg(sgl, sg, nents, j) {
|
|
sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
|
|
sg->offset, sg->length, per_page_flush);
|
|
if (sg->dma_address == DMA_MAPPING_ERROR)
|
|
return -EIO;
|
|
sg->dma_length = sg->length;
|
|
}
|
|
|
|
return nents;
|
|
}
|
|
|
|
static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
|
|
int nents, enum dma_data_direction dir, unsigned long attrs)
|
|
{
|
|
flush_page_for_dma(0);
|
|
return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
|
|
}
|
|
|
|
static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
|
|
int nents, enum dma_data_direction dir, unsigned long attrs)
|
|
{
|
|
return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
|
|
}
|
|
|
|
static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
|
|
size_t len, enum dma_data_direction dir, unsigned long attrs)
|
|
{
|
|
struct iommu_struct *iommu = dev->archdata.iommu;
|
|
unsigned int busa = dma_addr & PAGE_MASK;
|
|
unsigned long off = dma_addr & ~PAGE_MASK;
|
|
unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
|
|
unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
|
|
unsigned int i;
|
|
|
|
BUG_ON(busa < iommu->start);
|
|
for (i = 0; i < npages; i++) {
|
|
iopte_val(iommu->page_table[ioptex + i]) = 0;
|
|
iommu_invalidate_page(iommu->regs, busa);
|
|
busa += PAGE_SIZE;
|
|
}
|
|
bit_map_clear(&iommu->usemap, ioptex, npages);
|
|
}
|
|
|
|
static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
|
|
int nents, enum dma_data_direction dir, unsigned long attrs)
|
|
{
|
|
struct scatterlist *sg;
|
|
int i;
|
|
|
|
for_each_sg(sgl, sg, nents, i) {
|
|
sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
|
|
attrs);
|
|
sg->dma_address = 0x21212121;
|
|
}
|
|
}
|
|
|
|
#ifdef CONFIG_SBUS
|
|
static void *sbus_iommu_alloc(struct device *dev, size_t len,
|
|
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
|
|
{
|
|
struct iommu_struct *iommu = dev->archdata.iommu;
|
|
unsigned long va, addr, page, end, ret;
|
|
iopte_t *iopte = iommu->page_table;
|
|
iopte_t *first;
|
|
int ioptex;
|
|
|
|
/* XXX So what is maxphys for us and how do drivers know it? */
|
|
if (!len || len > 256 * 1024)
|
|
return NULL;
|
|
|
|
len = PAGE_ALIGN(len);
|
|
va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
|
|
if (va == 0)
|
|
return NULL;
|
|
|
|
addr = ret = sparc_dma_alloc_resource(dev, len);
|
|
if (!addr)
|
|
goto out_free_pages;
|
|
|
|
BUG_ON((va & ~PAGE_MASK) != 0);
|
|
BUG_ON((addr & ~PAGE_MASK) != 0);
|
|
BUG_ON((len & ~PAGE_MASK) != 0);
|
|
|
|
/* page color = physical address */
|
|
ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
|
|
addr >> PAGE_SHIFT);
|
|
if (ioptex < 0)
|
|
panic("iommu out");
|
|
|
|
iopte += ioptex;
|
|
first = iopte;
|
|
end = addr + len;
|
|
while(addr < end) {
|
|
page = va;
|
|
{
|
|
pmd_t *pmdp;
|
|
pte_t *ptep;
|
|
|
|
if (viking_mxcc_present)
|
|
viking_mxcc_flush_page(page);
|
|
else if (viking_flush)
|
|
viking_flush_page(page);
|
|
else
|
|
__flush_page_to_ram(page);
|
|
|
|
pmdp = pmd_off_k(addr);
|
|
ptep = pte_offset_map(pmdp, addr);
|
|
|
|
set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
|
|
}
|
|
iopte_val(*iopte++) =
|
|
MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
|
|
addr += PAGE_SIZE;
|
|
va += PAGE_SIZE;
|
|
}
|
|
/* P3: why do we need this?
|
|
*
|
|
* DAVEM: Because there are several aspects, none of which
|
|
* are handled by a single interface. Some cpus are
|
|
* completely not I/O DMA coherent, and some have
|
|
* virtually indexed caches. The driver DMA flushing
|
|
* methods handle the former case, but here during
|
|
* IOMMU page table modifications, and usage of non-cacheable
|
|
* cpu mappings of pages potentially in the cpu caches, we have
|
|
* to handle the latter case as well.
|
|
*/
|
|
flush_cache_all();
|
|
iommu_flush_iotlb(first, len >> PAGE_SHIFT);
|
|
flush_tlb_all();
|
|
iommu_invalidate(iommu->regs);
|
|
|
|
*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
|
|
return (void *)ret;
|
|
|
|
out_free_pages:
|
|
free_pages(va, get_order(len));
|
|
return NULL;
|
|
}
|
|
|
|
static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
|
|
dma_addr_t busa, unsigned long attrs)
|
|
{
|
|
struct iommu_struct *iommu = dev->archdata.iommu;
|
|
iopte_t *iopte = iommu->page_table;
|
|
struct page *page = virt_to_page(cpu_addr);
|
|
int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
|
|
unsigned long end;
|
|
|
|
if (!sparc_dma_free_resource(cpu_addr, len))
|
|
return;
|
|
|
|
BUG_ON((busa & ~PAGE_MASK) != 0);
|
|
BUG_ON((len & ~PAGE_MASK) != 0);
|
|
|
|
iopte += ioptex;
|
|
end = busa + len;
|
|
while (busa < end) {
|
|
iopte_val(*iopte++) = 0;
|
|
busa += PAGE_SIZE;
|
|
}
|
|
flush_tlb_all();
|
|
iommu_invalidate(iommu->regs);
|
|
bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
|
|
|
|
__free_pages(page, get_order(len));
|
|
}
|
|
#endif
|
|
|
|
static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
|
|
#ifdef CONFIG_SBUS
|
|
.alloc = sbus_iommu_alloc,
|
|
.free = sbus_iommu_free,
|
|
#endif
|
|
.map_page = sbus_iommu_map_page_gflush,
|
|
.unmap_page = sbus_iommu_unmap_page,
|
|
.map_sg = sbus_iommu_map_sg_gflush,
|
|
.unmap_sg = sbus_iommu_unmap_sg,
|
|
};
|
|
|
|
static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
|
|
#ifdef CONFIG_SBUS
|
|
.alloc = sbus_iommu_alloc,
|
|
.free = sbus_iommu_free,
|
|
#endif
|
|
.map_page = sbus_iommu_map_page_pflush,
|
|
.unmap_page = sbus_iommu_unmap_page,
|
|
.map_sg = sbus_iommu_map_sg_pflush,
|
|
.unmap_sg = sbus_iommu_unmap_sg,
|
|
};
|
|
|
|
void __init ld_mmu_iommu(void)
|
|
{
|
|
if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
|
|
dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
|
|
ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
|
|
} else {
|
|
dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
|
|
ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
|
|
}
|
|
}
|