diff --git a/zshrc b/zshrc
index 33a08ce..d4b0061 100644
--- a/zshrc
+++ b/zshrc
@@ -36,12 +36,14 @@ fi
 # GPG/SSH
 ## Void: Install pcsclite, pcsc-ccid, gnupg2-scdaemon
 ## Yubico openpgp: https://support.yubico.com/support/solutions/articles/15000006420-using-your-yubikey-with-openpgp
-unset SSH_AGENT_PID
-if [ "${gnupg_SSH_AUTH_SOCK_by:-0}" -ne $$ ]; then
-  export SSH_AUTH_SOCK="$(gpgconf --list-dirs agent-ssh-socket)"
+if [ -z "$SSH_CLIENT" ] || [ -z "$SSH_TTY" ]; then
+    unset SSH_AGENT_PID
+    if [ "${gnupg_SSH_AUTH_SOCK_by:-0}" -ne $$ ]; then
+      export SSH_AUTH_SOCK="$(gpgconf --list-dirs agent-ssh-socket)"
+    fi
+    export GPG_TTY=$(tty)
+    gpg-connect-agent updatestartuptty /bye >/dev/null
 fi
-export GPG_TTY=$(tty)
-gpg-connect-agent updatestartuptty /bye >/dev/null
 
 # VIM
 ## Reduce delay swithing between normal & insert mode