Non hai utilizzato THEN parola chiave e END IF .
Prova questo:
SET @ExistedRefID = NULL;
SELECT ReferenceID INTO @ExistedRefID
FROM filesrefrences
WHERE RefrenceKey= pRefrenceKey;
IF (@ExistedRefID IS NULL) THEN
INSERT INTO filesrefrences(RefrenceKey, RefrenceCount)
VALUES(pRefrenceKey,1);
SELECT LAST_INSERT_ID();
ELSE
UPDATE filesrefrences
SET RefrenceCount= RefrenceCount+1
WHERE ReferenceID= @ExistedRefID;
SELECT @ExistedRefID;
END IF;