/* Jenkins hash */
a = b = c = JHASH_INITVAL + sizeof(*key);
- a += key->asid;
+ a += key->asid + key->level + key->tg;
b += extract64(key->iova, 0, 32);
c += extract64(key->iova, 32, 32);
static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2)
{
- const SMMUIOTLBKey *k1 = v1;
- const SMMUIOTLBKey *k2 = v2;
+ SMMUIOTLBKey *k1 = (SMMUIOTLBKey *)v1, *k2 = (SMMUIOTLBKey *)v2;
- return (k1->asid == k2->asid) && (k1->iova == k2->iova);
+ return (k1->asid == k2->asid) && (k1->iova == k2->iova) &&
+ (k1->level == k2->level) && (k1->tg == k2->tg);
}
-SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova)
+SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova,
+ uint8_t tg, uint8_t level)
{
- SMMUIOTLBKey key = {.asid = asid, .iova = iova};
+ SMMUIOTLBKey key = {.asid = asid, .iova = iova, .tg = tg, .level = level};
return key;
}
SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg,
- hwaddr iova)
+ SMMUTransTableInfo *tt, hwaddr iova)
{
- SMMUIOTLBKey key = smmu_get_iotlb_key(cfg->asid, iova);
- SMMUTLBEntry *entry = g_hash_table_lookup(bs->iotlb, &key);
+ uint8_t tg = (tt->granule_sz - 10) / 2;
+ uint8_t inputsize = 64 - tt->tsz;
+ uint8_t stride = tt->granule_sz - 3;
+ uint8_t level = 4 - (inputsize - 4) / stride;
+ SMMUTLBEntry *entry = NULL;
+
+ while (level <= 3) {
+ uint64_t subpage_size = 1ULL << level_shift(level, tt->granule_sz);
+ uint64_t mask = subpage_size - 1;
+ SMMUIOTLBKey key;
+
+ key = smmu_get_iotlb_key(cfg->asid, iova & ~mask, tg, level);
+ entry = g_hash_table_lookup(bs->iotlb, &key);
+ if (entry) {
+ break;
+ }
+ level++;
+ }
if (entry) {
cfg->iotlb_hits++;
void smmu_iotlb_insert(SMMUState *bs, SMMUTransCfg *cfg, SMMUTLBEntry *new)
{
SMMUIOTLBKey *key = g_new0(SMMUIOTLBKey, 1);
+ uint8_t tg = (new->granule - 10) / 2;
if (g_hash_table_size(bs->iotlb) >= SMMU_IOTLB_MAX_SIZE) {
smmu_iotlb_inv_all(bs);
}
- *key = smmu_get_iotlb_key(cfg->asid, new->entry.iova);
- trace_smmu_iotlb_insert(cfg->asid, new->entry.iova);
+ *key = smmu_get_iotlb_key(cfg->asid, new->entry.iova, tg, new->level);
+ trace_smmu_iotlb_insert(cfg->asid, new->entry.iova, tg, new->level);
g_hash_table_insert(bs->iotlb, key, new);
}
return SMMU_IOTLB_ASID(*iotlb_key) == asid;
}
-inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova)
+static gboolean smmu_hash_remove_by_asid_iova(gpointer key, gpointer value,
+ gpointer user_data)
{
- SMMUIOTLBKey key = smmu_get_iotlb_key(asid, iova);
+ SMMUTLBEntry *iter = (SMMUTLBEntry *)value;
+ IOMMUTLBEntry *entry = &iter->entry;
+ SMMUIOTLBPageInvInfo *info = (SMMUIOTLBPageInvInfo *)user_data;
+ SMMUIOTLBKey iotlb_key = *(SMMUIOTLBKey *)key;
+
+ if (info->asid >= 0 && info->asid != SMMU_IOTLB_ASID(iotlb_key)) {
+ return false;
+ }
+ return (info->iova & ~entry->addr_mask) == entry->iova;
+}
+
+inline void smmu_iotlb_inv_iova(SMMUState *s, int asid, dma_addr_t iova)
+{
+ SMMUIOTLBPageInvInfo info = {.asid = asid, .iova = iova};
trace_smmu_iotlb_inv_iova(asid, iova);
- g_hash_table_remove(s->iotlb, &key);
+ g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid_iova, &info);
}
inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid)
baseaddr = extract64(tt->ttb, 0, 48);
baseaddr &= ~indexmask;
- tlbe->entry.iova = iova;
- tlbe->entry.addr_mask = (1 << granule_sz) - 1;
-
while (level <= 3) {
uint64_t subpage_size = 1ULL << level_shift(level, granule_sz);
uint64_t mask = subpage_size - 1;
goto error;
}
- tlbe->entry.translated_addr = gpa + (iova & mask);
+ tlbe->entry.translated_addr = gpa;
+ tlbe->entry.iova = iova & ~mask;
+ tlbe->entry.addr_mask = mask;
tlbe->entry.perm = PTE_AP_TO_PERM(ap);
tlbe->level = level;
tlbe->granule = granule_sz;
page_mask = (1ULL << (tt->granule_sz)) - 1;
aligned_addr = addr & ~page_mask;
- cached_entry = smmu_iotlb_lookup(bs, cfg, aligned_addr);
+ cached_entry = smmu_iotlb_lookup(bs, cfg, tt, aligned_addr);
if (cached_entry) {
if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & IOMMU_WO)) {
status = SMMU_TRANS_ERROR;
case SMMU_TRANS_SUCCESS:
entry.perm = flag;
entry.translated_addr = cached_entry->entry.translated_addr +
- (addr & page_mask);
+ (addr & cached_entry->entry.addr_mask);
entry.addr_mask = cached_entry->entry.addr_mask;
trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr,
entry.translated_addr, entry.perm);
trace_smmuv3_cmdq_tlbi_nh_vaa(vmid, addr);
smmuv3_inv_notifiers_iova(bs, -1, addr);
- smmu_iotlb_inv_all(bs);
+ smmu_iotlb_inv_iova(bs, -1, addr);
break;
}
case SMMU_CMD_TLBI_NH_VA:
typedef struct SMMUIOTLBKey {
uint64_t iova;
uint16_t asid;
+ uint8_t tg;
+ uint8_t level;
} SMMUIOTLBKey;
typedef struct SMMUState {
#define SMMU_IOTLB_MAX_SIZE 256
-SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg, hwaddr iova);
+SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg,
+ SMMUTransTableInfo *tt, hwaddr iova);
void smmu_iotlb_insert(SMMUState *bs, SMMUTransCfg *cfg, SMMUTLBEntry *entry);
-SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova);
+SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova,
+ uint8_t tg, uint8_t level);
void smmu_iotlb_inv_all(SMMUState *s);
void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid);
-void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova);
+void smmu_iotlb_inv_iova(SMMUState *s, int asid, dma_addr_t iova);
/* Unmap the range of all the notifiers registered to any IOMMU mr */
void smmu_inv_notifiers_all(SMMUState *s);