diff --git a/common-session.c b/common-session.c
index 90129b42f7c6fca380d5908f042cd882e5067eda..a90673fb2300af6610441e42c965c3a6ee9107f4 100644
--- a/common-session.c
+++ b/common-session.c
@@ -60,7 +60,6 @@ void common_session_init(int sock_in, int sock_out) {
 	ses.maxfd = MAX(sock_in, sock_out);
 
 	now = monotonic_now();
-	ses.connect_time = now;
 	ses.last_packet_time_keepalive_recv = now;
 	ses.last_packet_time_idle = now;
 	ses.last_packet_time_any_sent = 0;
@@ -415,10 +414,6 @@ static void checktimeouts() {
 	time_t now;
 	now = monotonic_now();
 	
-	if (now - ses.connect_time >= AUTH_TIMEOUT) {
-			dropbear_close("Timeout before auth");
-	}
-
 	/* we can't rekey if we haven't done remote ident exchange yet */
 	if (ses.remoteident == NULL) {
 		return;
diff --git a/session.h b/session.h
index 16a6e37c1dd9f3bab17c45142808239f442c1c3f..548dabd6d7ed9744b7ed9b82abc6fb7aa89ac9d0 100644
--- a/session.h
+++ b/session.h
@@ -104,11 +104,6 @@ struct sshsession {
 	/* Is it a client or server? */
 	unsigned char isserver;
 
-	time_t connect_time; /* time the connection was established
-							(cleared after auth once we're not
-							respecting AUTH_TIMEOUT any more).
-							A monotonic time, not realworld */
-
 	int sock_in;
 	int sock_out;
 
@@ -221,6 +216,11 @@ struct serversession {
 	/* The resolved remote address, used for lastlog etc */
 	char *remotehost;
 
+	time_t connect_time; /* time the connection was established
+							(cleared after auth once we're not
+							respecting AUTH_TIMEOUT any more).
+							A monotonic time, not realworld */
+
 #ifdef USE_VFORK
 	pid_t server_pid;
 #endif
diff --git a/svr-auth.c b/svr-auth.c
index 9051d85fc1f5c7b7ecb323727328b8634624b5ff..89760ef9e5db52320acb21f32d4bcf4a23e92394 100644
--- a/svr-auth.c
+++ b/svr-auth.c
@@ -392,8 +392,7 @@ void send_msg_userauth_success() {
 	/* authdone must be set after encrypt_packet() for 
 	 * delayed-zlib mode */
 	ses.authstate.authdone = 1;
-	ses.connect_time = 0;
-
+	svr_ses.connect_time = 0;
 
 	if (ses.authstate.pw_uid == 0) {
 		ses.allowprivport = 1;
diff --git a/svr-session.c b/svr-session.c
index 90d3414fc2728b35da0cdcbde11deec3d567ed49..4d3c058a1c503273b4500f2c6f05a1e3cae82807 100644
--- a/svr-session.c
+++ b/svr-session.c
@@ -80,12 +80,22 @@ svr_session_cleanup(void)
 	svr_pubkey_options_cleanup();
 }
 
+static void
+svr_sessionloop() {
+	if (svr_ses.connect_time != 0 
+		&& monotonic_now() - svr_ses.connect_time >= AUTH_TIMEOUT) {
+		dropbear_close("Timeout before auth");
+	}
+}
+
 void svr_session(int sock, int childpipe) {
 	char *host, *port;
 	size_t len;
 
 	common_session_init(sock, sock);
 
+	svr_ses.connect_time = monotonic_now();;
+
 	/* Initialise server specific parts of the session */
 	svr_ses.childpipe = childpipe;
 #ifdef USE_VFORK
@@ -126,7 +136,7 @@ void svr_session(int sock, int childpipe) {
 
 	/* Run the main for loop. NULL is for the dispatcher - only the client
 	 * code makes use of it */
-	session_loop(NULL);
+	session_loop(svr_sessionloop);
 
 	/* Not reached */