There are a bunch of properties on AdvancedTabItem for Brushes to use in various states and our default template uses those, however code in AdvancedTabControl.PrepareContainerForItemOverride binds those to related Tab* properties set on the AdvancedTabControl itself. That makes it really easy to customize a look of an entire tab control but won't work for individual tabs needing to be different.
So you'd need to make a custom style/template for the tab items. Below is a quick (and ugly) sample of customizing the tab item template. Instead of using fixed color values like I do, you could use bindings in those Setter values.
xmlns:docking="http://schemas.actiprosoftware.com/winfx/xaml/docking"
xmlns:shared="http://schemas.actiprosoftware.com/winfx/xaml/shared"
xmlns:sampleBrowser="clr-namespace:ActiproSoftware.SampleBrowser"
xmlns:system="clr-namespace:System;assembly=mscorlib"
...
<BooleanToVisibilityConverter x:Key="BooleanToVisibilityConverter" />
<shared:ConditionalConverter x:Key="BooleanToOpacityConverter">
<shared:ConditionalConverter.TrueValue>
<system:Double>1</system:Double>
</shared:ConditionalConverter.TrueValue>
<shared:ConditionalConverter.FalseValue>
<system:Double>0</system:Double>
</shared:ConditionalConverter.FalseValue>
</shared:ConditionalConverter>
<docking:TitleConverter x:Key="TitleConverter" />
<shared:ThicknessConverter x:Key="ThicknessConverter" />
...
<docking:DockSite x:Name="dockSite">
<docking:Workspace>
<docking:TabbedMdiHost>
<docking:TabbedMdiHost.TabItemContainerStyle>
<Style TargetType="docking:AdvancedTabItem">
<Setter Property="Header" Value="{Binding TabTextResolved, Converter={StaticResource TitleConverter}}" />
<Setter Property="HeaderTemplate">
<Setter.Value>
<DataTemplate>
<TextBlock Text="{Binding}" TextTrimming="None" TextWrapping="NoWrap" VerticalAlignment="Center" />
</DataTemplate>
</Setter.Value>
</Setter>
<Setter Property="Template">
<Setter.Value>
<ControlTemplate TargetType="docking:AdvancedTabItem">
<Grid Background="Transparent">
<docking:TabItemBorder x:Name="TabChrome" IsSelected="{TemplateBinding IsSelected}"
TabStripPlacement="{TemplateBinding TabStripPlacement}"
NearSlantExtent="{TemplateBinding NearSlantExtent}" FarSlantExtent="{TemplateBinding FarSlantExtent}"
BorderThickness="{TemplateBinding BorderThickness}" UntintedBorderBrush="{TemplateBinding BorderBrush}"
UntintedBackground="{TemplateBinding Background}" CornerRadius="{TemplateBinding CornerRadius}"
TintColor="{TemplateBinding TintColor}" FlashColor="{TemplateBinding FlashColor}" FlashMode="{TemplateBinding FlashMode}">
<Grid>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="*" />
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="Auto" />
</Grid.ColumnDefinitions>
<Image AutomationProperties.Name="Image" Width="16" Height="16" Stretch="Uniform" Source="{TemplateBinding ImageSource}"
Margin="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=Padding, Converter={StaticResource ThicknessConverter}, ConverterParameter='Left'}"
Visibility="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=IsImageVisible, Converter={StaticResource BooleanToVisibilityConverter}}"
VerticalAlignment="Center" SnapsToDevicePixels="True" />
<shared:PixelSnapper Grid.Column="1" VerticalRoundMode="CeilingToEven">
<ContentPresenter x:Name="ContentPresenter" Margin="{TemplateBinding Padding}"
Content="{TemplateBinding Header}" ContentTemplate="{TemplateBinding HeaderTemplate}"
TextElement.Foreground="{TemplateBinding Foreground}"
HorizontalAlignment="{TemplateBinding HorizontalContentAlignment}" VerticalAlignment="{TemplateBinding VerticalContentAlignment}">
<ContentPresenter.Resources>
<Style TargetType="TextBlock">
<Setter Property="TextTrimming" Value="CharacterEllipsis" />
<Setter Property="TextWrapping" Value="NoWrap" />
</Style>
</ContentPresenter.Resources>
</ContentPresenter>
</shared:PixelSnapper>
<shared:PixelSnapper Grid.Column="2" VerticalRoundMode="CeilingToEven">
<ContentControl x:Name="ContextContentControl" IsTabStop="False"
Content="{TemplateBinding ContextContent}" ContentTemplate="{TemplateBinding ContextContentTemplate}"
TextElement.Foreground="{Binding ElementName=ContentPresenter, Path=(TextElement.Foreground)}" VerticalAlignment="Center" />
</shared:PixelSnapper>
<StackPanel x:Name="ButtonPanel" Grid.Column="3" Orientation="Horizontal" HorizontalAlignment="Center" VerticalAlignment="Center">
<Button x:Name="ToggleLayoutKindButton" ContentTemplate="{TemplateBinding PinButtonContentTemplate}" Style="{TemplateBinding EmbeddedButtonStyle}"
Command="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=ToggleLayoutKindCommand}"
Foreground="{Binding ElementName=ContentPresenter, Path=(TextElement.Foreground)}"
Visibility="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=HasToggleLayoutKindButton, Converter={StaticResource BooleanToVisibilityConverter}}"
Opacity="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=IsToggleLayoutKindButtonVisible, Converter={StaticResource BooleanToOpacityConverter}}"
ToolTip="{docking:SRExtension UIAdvancedTabItemToggleLayoutKindButtonToolTip}" AutomationProperties.Name="Toggle Layout Kind" />
<Button x:Name="CloseButton" ContentTemplate="{TemplateBinding CloseButtonContentTemplate}" Style="{TemplateBinding EmbeddedButtonStyle}"
Command="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=CloseCommand}"
Foreground="{Binding ElementName=ContentPresenter, Path=(TextElement.Foreground)}"
Visibility="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=HasCloseButton, Converter={StaticResource BooleanToVisibilityConverter}}"
Opacity="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=IsCloseButtonVisible, Converter={StaticResource BooleanToOpacityConverter}}"
ToolTip="{docking:SRExtension UIAdvancedTabItemCloseButtonToolTip}" AutomationProperties.Name="Close" />
</StackPanel>
</Grid>
</docking:TabItemBorder>
</Grid>
<ControlTemplate.Triggers>
<Trigger Property="IsReadOnly" Value="True">
<Setter TargetName="ContextContentControl" Property="ContentTemplate" Value="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=ReadOnlyContextContentTemplate}" />
</Trigger>
<Trigger Property="LayoutKind" Value="Pinned">
<Setter TargetName="ToggleLayoutKindButton" Property="ContentTemplate" Value="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=UnpinButtonContentTemplate}" />
</Trigger>
<Trigger Property="HighlightKind" Value="PointerOver">
<Setter TargetName="TabChrome" Property="UntintedBackground" Value="LightGreen" />
<Setter TargetName="TabChrome" Property="UntintedBorderBrush" Value="Green" />
<Setter TargetName="ContentPresenter" Property="TextElement.Foreground" Value="Black" />
</Trigger>
<Trigger Property="HighlightKind" Value="ActiveSelected">
<Setter TargetName="TabChrome" Property="UntintedBackground" Value="LightBlue" />
<Setter TargetName="TabChrome" Property="UntintedBorderBrush" Value="Blue" />
<Setter TargetName="ContentPresenter" Property="TextElement.Foreground" Value="Black" />
</Trigger>
<Trigger Property="HighlightKind" Value="InactiveSelected">
<Setter TargetName="TabChrome" Property="UntintedBackground" Value="Transparent" />
<Setter TargetName="TabChrome" Property="UntintedBorderBrush" Value="#303030" />
<Setter TargetName="ContentPresenter" Property="TextElement.Foreground" Value="#303030" />
</Trigger>
</ControlTemplate.Triggers>
</ControlTemplate>
</Setter.Value>
</Setter>
</Style>
</docking:TabbedMdiHost.TabItemContainerStyle>
<docking:TabbedMdiContainer>
<docking:DocumentWindow Title="Document1.txt">
<TextBox BorderThickness="0" />
</docking:DocumentWindow>
<docking:DocumentWindow Title="Document2.txt">
<TextBox BorderThickness="0" />
</docking:DocumentWindow>
<docking:DocumentWindow Title="Document3.txt">
<TextBox BorderThickness="0" />
</docking:DocumentWindow>
</docking:TabbedMdiContainer>
</docking:TabbedMdiHost>
</docking:Workspace>
</docking:DockSite>