[Phanx]

Originally Posted by eiszeit ... maybe adding the IsInInstance()-Variant when I have an idea how I do that exactly.

Just do like I said, and move the IsInInstance check out of the filter function. If you don't already have a frame in that file that can listen for events (I have no idea if that's a whole file or just a snippet you posted) just create one.

lib.AurasCustomFilter = {
    target = function(icons, unit, icon, name, _, _, _, dtype, _, _, caster, isStealable, shouldConsolidate, spellID, _, isBossAura, isCastByPlayer)
        local _, instanceType = IsInInstance()

-->

local instanceType

local f = CreateFrame("Frame")
f:RegisterEvent("PLAYER_ENTERING_WORLD")
f:SetScript("OnEvent", function()
    instanceType, instanceType = IsInInstance()
    -- yes, duplication is intentional, it avoids having to name another variable
    -- the second one overwrites the first one
end)

lib.AurasCustomFilter = {
    target = function(icons, unit, icon, name, _, _, _, dtype, _, _, caster, isStealable, shouldConsolidate, spellID, _, isBossAura, isCastByPlayer)