Following sample wxs file demonstrates how to use CustomAction to invoke InstallUtil.exe to run your managed installer class.
<?xml version='1.0'?>
<Wix xmlns='http://schemas.microsoft.com/wix/2003/01/wi'>
<Product Id='12345678-1234-1234-1234-123456789012' Name='Test Package' Language='1033'
Version='1.0.0.0' Manufacturer='Microsoft Corporation'>
<Package Id='12345678-1234-1234-1234-123456789012'
Description='My first Windows Installer package based on InstallUtil.exe'
Comments='This is my first attempt at creating a Windows Installer database'
Manufacturer='Microsoft Corporation' InstallerVersion='200' Compressed='yes' />
<Media Id='1' Cabinet='product.cab' EmbedCab='yes' />
<Directory Id='SourceDir' Name='SourceDir'>
<Directory Id='ProgramFilesFolder' Name='PFDir'>
<Directory Id='TestDir' Name='TestDir' LongName='Test Program'>
<Component Id='MyComponent' Guid='12345678-1234-1234-1234-123456789012'>
<File Id='tryInstall' Name="try" LongName='tryInstall.exe' DiskId='1' src='tryInstall.exe' />
</Component>
</Directory>
</Directory>
</Directory>
<Feature Id='MyFeature' Title='My 1st Feature' Level='1'>
<ComponentRef Id='MyComponent' />
</Feature>
<InstallExecuteSequence>
<Custom Action='ManagedInstall' After="InstallFinalize" />
</InstallExecuteSequence>
<CustomAction Id="ManagedInstall"
Directory='TestDir'
ExeCommand='"[WindowsFolder]Microsoft.NETFrameworkv2.0.50727installUtil.exe" /LogToConsole=false tryInstall.exe'
Return='check'>
</CustomAction>
</Product>
</Wix>