PowerShell is very structured, but there are exception, removing an alias is one of many.

# Create a new alias
New-Alias -Name Test -Value TestValue
# Check if the alias was created
Get-Alias Test
# Remove that sucker
# Naturally in PowerShell you'd think it is remove-alias, but it is not
Remove-Item alias:Test