Just look at one square 2x2 :)

a = []
for i in range(4):
    a.append(list(map(int, input().split())))
b = [0, 0, 0, 0]
b[a[0][0] - 1] += 1
b[a[0][1] - 1] += 1
b[a[1][1] - 1] += 1
b[a[1][0] - 1] += 1
m, n = 0, 0
for i in range(4):
    if b[i] > m:
        m = b[i]
        n = i
s = 0
for i in range(4):
    if i != n:
        s += min((n-i) % 4, (i-n) % 4) * b[i]
print(s)

My code works at O(8)
time: 0.093
Python