[SDPhantom]

Here's some code I'm posting separately since it spans past the realm of just quest rewards. This will change the color of any item still usable, but differs from the preferred armor type of your character to a blue tint. This doesn't look at stats, just whether it's cloth/leather/mail/plate and compares it to what type you should be wearing for your class.

Combined with what the default UI already does, this means red-tinted items are unusable while blue-tinted items are still usable, but not the best type to do so. This affects mail attachments, vendors, quest info, and trade windows, the same areas the default UI handles.

local UpdateItemButtonUsableColor; do
    local ItemClassLookup={GetAuctionItemClasses()}; for i,j in ipairs(ItemClassLookup) do
        local sub={[0]=i,GetAuctionItemSubClasses(i)};
        ItemClassLookup[j]=sub;
        for k,l in ipairs(sub) do sub[l]=k; end
    end
 
    local EquipLocations={
        INVTYPE_HEAD        =true;
        INVTYPE_SHOULDER    =true;
        INVTYPE_CHEST       =true;
        INVTYPE_ROBE        =true;
        INVTYPE_WAIST       =true;
        INVTYPE_LEGS        =true;
        INVTYPE_FEET        =true;
        INVTYPE_WRIST       =true;
        INVTYPE_HAND        =true;
    }
 
    local ArmorPreferenceData={
        CLOTH       ={2,2};
        LEATHER     ={3,3};
        LEATHER_MAIL    ={3,4};
        MAIL_PLATE  ={4,5};
        PLATE       ={5,5};
    }
    local ClassArmorPreference={
--      Cloth
        MAGE        =ArmorPreferenceData.CLOTH;
        PRIEST      =ArmorPreferenceData.CLOTH;
        WARLOCK     =ArmorPreferenceData.CLOTH;
 
--      Leather
        DRUID       =ArmorPreferenceData.LEATHER;
        ROGUE       =ArmorPreferenceData.LEATHER;
 
--      Leather/Mail
        HUNTER      =ArmorPreferenceData.LEATHER_MAIL;
        SHAMAN      =ArmorPreferenceData.LEATHER_MAIL;
 
--      Mail/Plate
        WARRIOR     =ArmorPreferenceData.MAIL_PLATE;
        PALADIN     =ArmorPreferenceData.MAIL_PLATE;
 
--      Plate
        DEATHKNIGHT =ArmorPreferenceData.PLATE;
    };
 
    local PlayerClass;
    UpdateItemButtonUsableColor=function(btn,item,flag)
        if not PlayerClass then PlayerClass=select(2,UnitClass("player")); end
        if not item then return; end
 
        local lvlid=(UnitLevel("player")<40 and 1 or 2);
        local class,sub,_,loc=select(6,GetItemInfo(item));
        local classdata=ItemClassLookup[class];
 
        if  EquipLocations[loc]
        and classdata[sub]>1 and classdata[sub]<6
        and sub~=classdata[ClassArmorPreference[PlayerClass][lvlid]]
        then
            local int=1;
            if bit.band(flag,16)>0 then int=0.5; end
 
            if bit.band(flag,1)>0 then SetItemButtonTextureVertexColor(btn,0,0,int); end
            if bit.band(flag,2)>0 then SetItemButtonNameFrameVertexColor(btn,0,0,int); end
            if bit.band(flag,4)>0 then SetItemButtonSlotVertexColor(btn,0,0,int); end
            if bit.band(flag,8)>0 then SetItemButtonNormalTextureVertexColor(btn,0,0,int); end
        end
    end;
end
 
hooksecurefunc("OpenMailFrame_UpdateButtonPositions",function()
    for i=1,ATTACHMENTS_MAX_RECEIVE do
        if select(5,GetInboxItem(InboxFrame.openMailID,i)) then
            UpdateItemButtonUsableColor(_G["OpenMailAttachmentButton"..i],GetInboxItemLink(InboxFrame.openMailID,i),1);
        end
    end
end);
 
hooksecurefunc("MerchantFrame_UpdateMerchantInfo",function()
    local numitems=GetMerchantNumItems();
    for i=1,MERCHANT_ITEMS_PER_PAGE do
        local id=((MerchantFrame.page-1)*MERCHANT_ITEMS_PER_PAGE)+i;
        if id>numitems then break; end
 
        local name,_,_,_,avail,usable=GetMerchantItemInfo(id);
        if name and usable then--   usable may return true even when it's not an item
            local mercbtn=_G["MerchantItem"..i];
            local itembtn=_G["MerchantItem"..i.."ItemButton"];
            local link=GetMerchantItemLink(id);
 
            if avail==0 then--  -1 is infinite
                UpdateItemButtonUsableColor(mercbtn,link,22);
                UpdateItemButtonUsableColor(itembtn,link,25);
            else
                UpdateItemButtonUsableColor(mercbtn,link,6);
                UpdateItemButtonUsableColor(itembtn,link,9);
            end
        end
    end
end);
 
hooksecurefunc("QuestInfo_ShowRewards",function()
    local isquestlog=QuestInfoFrame.questLog;
    local numchoice=(isquestlog and GetNumQuestLogChoices() or GetNumQuestChoices());
    local numreward=(isquestlog and GetNumQuestLogRewards() or GetNumQuestRewards());
 
    for i=1,numchoice do
        if select(5,_G[isquestlog and "GetQuestLogChoiceInfo" or "GetQuestItemInfo"](isquestlog and i or "choice",i)) then
            UpdateItemButtonUsableColor(
                _G["QuestInfoItem"..i]
                ,_G[isquestlog and "GetQuestLogItemLink" or "GetQuestItemLink"]("choice",i)
                ,3
            );
        end
    end
 
    for i=1,numreward do
        if select(5,_G[isquestlog and "GetQuestLogRewardInfo" or "GetQuestItemInfo"](isquestlog and i or "reward",i)) then
            UpdateItemButtonUsableColor(
                _G["QuestInfoItem"..(numchoice+i)]
                ,_G[isquestlog and "GetQuestLogItemLink" or "GetQuestItemLink"]("reward",i)
                ,3
            );
        end
    end
end);
QUEST_TEMPLATE_DETAIL2.elements[13]=QuestInfo_ShowRewards;
QUEST_TEMPLATE_LOG.elements[28]=QuestInfo_ShowRewards;
QUEST_TEMPLATE_REWARD.elements[7]=QuestInfo_ShowRewards;
QUEST_TEMPLATE_MAP2.elements[1]=QuestInfo_ShowRewards;
 
hooksecurefunc("TradeFrame_UpdateTargetItem",function(id)
    if select(5,GetTradeTargetItemInfo(id)) then
        local link=GetTradeTargetItemLink(id);
        UpdateItemButtonUsableColor(_G["TradeRecipientItem"..id.."ItemButton"],link,1);
        UpdateItemButtonUsableColor(_G["TradeRecipientItem"..id],link,6);
    end
end);

Edit1: Fixed problems with random items like cloaks misreading as cloth armor. Now checks on equip location too before deciding whether to tint the item.
Edit2: Fixed an error with the QuestFrame sometimes not having the items ready when calling the QuestInfo_ShowRewards() function.