Maybe, it can help to detect wrong codes:
1 - 2, 2 - 1, 0 - 3, 3 - 0

See the example below.

Also you should scan the encoder wery fast.
E.g. I use T0 in mode 3, which give me an interrupt every 256 cycles (= 278µs).


Peter



JogA bit p1.1
JogB bit p2.7

JogOld equ 30h
JogCount equ 31h

intt0:
push psw
push acc
setb f0
mov a, JogOld
intt0a: mov c, JogA
rlc a
mov c, JogB
rlc a
mov JogOld, a
anl a, #0Fh
add a, #intt0d - intt0b
movc a, @a+pc
intt0b: cjne a, #2, intt0c
mov a, JogOld
rrc a
rrc a
jbc f0, intt0a ; read again 1 time
clr a
intt0c: add a, JogCount
mov JogCount, a
pop acc
pop psw
reti
intt0d:

; 4 phase = 4 steps:
; CW: 0 - 1 - 3 - 2 - 0
; CCW: 0 - 2 - 3 - 1 - 0
; again: 0 - 3, 3 - 0, 1 - 2, 2 - 1
db 0, 1, -1, 2, -1, 0, 2, 1
db 1, 2, 0, -1, 2, -1, 1, 0