da varbinary a varchar senza master.dbo.fn_varbintohexstr

Per md5 e sha1 puoi utilizzare hashbytes . Per ottenere base64 puoi creare un udf che esegua la conversione e utilizzarlo nella colonna calcolata.

Funzione BinToBase64:

create function BinToBase64(@Bin varbinary(max)) returns varchar(max) as
begin
  return CAST(N'' AS XML).value('xs:base64Binary(xs:hexBinary(sql:variable("@Bin")))', 'VARCHAR(MAX)')
end

Funzione BinToHexString:

create function BinToHexString(@Bin varbinary(max)) returns varchar(max) as
begin
  return '0x' + cast('' as xml).value('xs:hexBinary(sql:variable("@Bin") )', 'varchar(max)'); 
end

Usa in questo modo:

create table TestTable 
(
  Col1 varbinary(max), 
  Col2 as dbo.BinToHexString(hashbytes('MD5', Col1)),
  Col3 as dbo.BinToHexString(hashbytes('SHA1', Col1)),
  Col4 as dbo.BinToBase64(Col1),
)
insert into TestTable values (12345)

select *
from TestTable

Colonna varbinary con vincolo univoco che utilizza hashbyte e una colonna uniqueidentifier

create table TestTable 
(
  ID uniqueidentifier default(newid()),
  Col1 varbinary(max), 
  Col2 as coalesce(hashbytes('MD5', Col1), cast(ID as varbinary(8000))) persisted
)

create unique index IX_TestTable_Col2 on TestTable(Col2)