[ObbleYeah]

I did this the other day too, you actually have to create a new texture correctly sized and parented to PlayerFrameBackground rather than modifying the frame itself - else, as you've noticed, the entire unit background is recoloured.

--
    if PlayerFrame:IsShown() and not bg then
        local _, class = UnitClass("player")
        local colour = RAID_CLASS_COLORS[class]
        local bg = PlayerFrame:CreateTexture(nil, "ARTWORK")
            
        bg:SetPoint("TOPLEFT", PlayerFrameBackground)
        bg:SetPoint("BOTTOMRIGHT", PlayerFrameBackground, 0, 22)
        bg:SetVertexColor(colour.r, colour.g, colour.b, .95)
        bg:SetTexture("texturepath")
    end