364 lines
8.3 KiB
C
364 lines
8.3 KiB
C
|
// SPDX-License-Identifier: GPL-2.0
|
||
|
|
/*
|
||
|
|
* csum_partial_copy - do IP checksumming and copy
|
||
|
|
*
|
||
|
|
* (C) Copyright 1996 Linus Torvalds
|
||
|
|
* accelerated versions (and 21264 assembly versions ) contributed by
|
||
|
|
* Rick Gorton <rick.gorton@alpha-processor.com>
|
||
|
|
*
|
||
|
|
* Don't look at this too closely - you'll go mad. The things
|
||
|
|
* we do for performance..
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <linux/types.h>
|
||
|
|
#include <linux/string.h>
|
||
|
|
#include <linux/uaccess.h>
|
||
|
|
#include <net/checksum.h>
|
||
|
|
|
||
|
|
|
||
|
|
#define ldq_u(x,y) \
|
||
|
|
__asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
|
||
|
|
|
||
|
|
#define stq_u(x,y) \
|
||
|
|
__asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
|
||
|
|
|
||
|
|
#define extql(x,y,z) \
|
||
|
|
__asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
|
||
|
|
|
||
|
|
#define extqh(x,y,z) \
|
||
|
|
__asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
|
||
|
|
|
||
|
|
#define mskql(x,y,z) \
|
||
|
|
__asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
|
||
|
|
|
||
|
|
#define mskqh(x,y,z) \
|
||
|
|
__asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
|
||
|
|
|
||
|
|
#define insql(x,y,z) \
|
||
|
|
__asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
|
||
|
|
|
||
|
|
#define insqh(x,y,z) \
|
||
|
|
__asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
|
||
|
|
|
||
|
|
#define __get_word(insn,x,ptr) \
|
||
|
|
({ \
|
||
|
|
long __guu_err; \
|
||
|
|
__asm__ __volatile__( \
|
||
|
|
"1: "#insn" %0,%2\n" \
|
||
|
|
"2:\n" \
|
||
|
|
EXC(1b,2b,%0,%1) \
|
||
|
|
: "=r"(x), "=r"(__guu_err) \
|
||
|
|
: "m"(__m(ptr)), "1"(0)); \
|
||
|
|
__guu_err; \
|
||
|
|
})
|
||
|
|
|
||
|
|
static inline unsigned short from64to16(unsigned long x)
|
||
|
|
{
|
||
|
|
/* Using extract instructions is a bit more efficient
|
||
|
|
than the original shift/bitmask version. */
|
||
|
|
|
||
|
|
union {
|
||
|
|
unsigned long ul;
|
||
|
|
unsigned int ui[2];
|
||
|
|
unsigned short us[4];
|
||
|
|
} in_v, tmp_v, out_v;
|
||
|
|
|
||
|
|
in_v.ul = x;
|
||
|
|
tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
|
||
|
|
|
||
|
|
/* Since the bits of tmp_v.sh[3] are going to always be zero,
|
||
|
|
we don't have to bother to add that in. */
|
||
|
|
out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
|
||
|
|
+ (unsigned long) tmp_v.us[2];
|
||
|
|
|
||
|
|
/* Similarly, out_v.us[2] is always zero for the final add. */
|
||
|
|
return out_v.us[0] + out_v.us[1];
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Ok. This isn't fun, but this is the EASY case.
|
||
|
|
*/
|
||
|
|
static inline unsigned long
|
||
|
|
csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
|
||
|
|
long len)
|
||
|
|
{
|
||
|
|
unsigned long checksum = ~0U;
|
||
|
|
unsigned long carry = 0;
|
||
|
|
|
||
|
|
while (len >= 0) {
|
||
|
|
unsigned long word;
|
||
|
|
if (__get_word(ldq, word, src))
|
||
|
|
return 0;
|
||
|
|
checksum += carry;
|
||
|
|
src++;
|
||
|
|
checksum += word;
|
||
|
|
len -= 8;
|
||
|
|
carry = checksum < word;
|
||
|
|
*dst = word;
|
||
|
|
dst++;
|
||
|
|
}
|
||
|
|
len += 8;
|
||
|
|
checksum += carry;
|
||
|
|
if (len) {
|
||
|
|
unsigned long word, tmp;
|
||
|
|
if (__get_word(ldq, word, src))
|
||
|
|
return 0;
|
||
|
|
tmp = *dst;
|
||
|
|
mskql(word, len, word);
|
||
|
|
checksum += word;
|
||
|
|
mskqh(tmp, len, tmp);
|
||
|
|
carry = checksum < word;
|
||
|
|
*dst = word | tmp;
|
||
|
|
checksum += carry;
|
||
|
|
}
|
||
|
|
return checksum;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* This is even less fun, but this is still reasonably
|
||
|
|
* easy.
|
||
|
|
*/
|
||
|
|
static inline unsigned long
|
||
|
|
csum_partial_cfu_dest_aligned(const unsigned long __user *src,
|
||
|
|
unsigned long *dst,
|
||
|
|
unsigned long soff,
|
||
|
|
long len)
|
||
|
|
{
|
||
|
|
unsigned long first;
|
||
|
|
unsigned long word, carry;
|
||
|
|
unsigned long lastsrc = 7+len+(unsigned long)src;
|
||
|
|
unsigned long checksum = ~0U;
|
||
|
|
|
||
|
|
if (__get_word(ldq_u, first,src))
|
||
|
|
return 0;
|
||
|
|
carry = 0;
|
||
|
|
while (len >= 0) {
|
||
|
|
unsigned long second;
|
||
|
|
|
||
|
|
if (__get_word(ldq_u, second, src+1))
|
||
|
|
return 0;
|
||
|
|
extql(first, soff, word);
|
||
|
|
len -= 8;
|
||
|
|
src++;
|
||
|
|
extqh(second, soff, first);
|
||
|
|
checksum += carry;
|
||
|
|
word |= first;
|
||
|
|
first = second;
|
||
|
|
checksum += word;
|
||
|
|
*dst = word;
|
||
|
|
dst++;
|
||
|
|
carry = checksum < word;
|
||
|
|
}
|
||
|
|
len += 8;
|
||
|
|
checksum += carry;
|
||
|
|
if (len) {
|
||
|
|
unsigned long tmp;
|
||
|
|
unsigned long second;
|
||
|
|
if (__get_word(ldq_u, second, lastsrc))
|
||
|
|
return 0;
|
||
|
|
tmp = *dst;
|
||
|
|
extql(first, soff, word);
|
||
|
|
extqh(second, soff, first);
|
||
|
|
word |= first;
|
||
|
|
mskql(word, len, word);
|
||
|
|
checksum += word;
|
||
|
|
mskqh(tmp, len, tmp);
|
||
|
|
carry = checksum < word;
|
||
|
|
*dst = word | tmp;
|
||
|
|
checksum += carry;
|
||
|
|
}
|
||
|
|
return checksum;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* This is slightly less fun than the above..
|
||
|
|
*/
|
||
|
|
static inline unsigned long
|
||
|
|
csum_partial_cfu_src_aligned(const unsigned long __user *src,
|
||
|
|
unsigned long *dst,
|
||
|
|
unsigned long doff,
|
||
|
|
long len,
|
||
|
|
unsigned long partial_dest)
|
||
|
|
{
|
||
|
|
unsigned long carry = 0;
|
||
|
|
unsigned long word;
|
||
|
|
unsigned long second_dest;
|
||
|
|
unsigned long checksum = ~0U;
|
||
|
|
|
||
|
|
mskql(partial_dest, doff, partial_dest);
|
||
|
|
while (len >= 0) {
|
||
|
|
if (__get_word(ldq, word, src))
|
||
|
|
return 0;
|
||
|
|
len -= 8;
|
||
|
|
insql(word, doff, second_dest);
|
||
|
|
checksum += carry;
|
||
|
|
stq_u(partial_dest | second_dest, dst);
|
||
|
|
src++;
|
||
|
|
checksum += word;
|
||
|
|
insqh(word, doff, partial_dest);
|
||
|
|
carry = checksum < word;
|
||
|
|
dst++;
|
||
|
|
}
|
||
|
|
len += 8;
|
||
|
|
if (len) {
|
||
|
|
checksum += carry;
|
||
|
|
if (__get_word(ldq, word, src))
|
||
|
|
return 0;
|
||
|
|
mskql(word, len, word);
|
||
|
|
len -= 8;
|
||
|
|
checksum += word;
|
||
|
|
insql(word, doff, second_dest);
|
||
|
|
len += doff;
|
||
|
|
carry = checksum < word;
|
||
|
|
partial_dest |= second_dest;
|
||
|
|
if (len >= 0) {
|
||
|
|
stq_u(partial_dest, dst);
|
||
|
|
if (!len) goto out;
|
||
|
|
dst++;
|
||
|
|
insqh(word, doff, partial_dest);
|
||
|
|
}
|
||
|
|
doff = len;
|
||
|
|
}
|
||
|
|
ldq_u(second_dest, dst);
|
||
|
|
mskqh(second_dest, doff, second_dest);
|
||
|
|
stq_u(partial_dest | second_dest, dst);
|
||
|
|
out:
|
||
|
|
checksum += carry;
|
||
|
|
return checksum;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* This is so totally un-fun that it's frightening. Don't
|
||
|
|
* look at this too closely, you'll go blind.
|
||
|
|
*/
|
||
|
|
static inline unsigned long
|
||
|
|
csum_partial_cfu_unaligned(const unsigned long __user * src,
|
||
|
|
unsigned long * dst,
|
||
|
|
unsigned long soff, unsigned long doff,
|
||
|
|
long len, unsigned long partial_dest)
|
||
|
|
{
|
||
|
|
unsigned long carry = 0;
|
||
|
|
unsigned long first;
|
||
|
|
unsigned long lastsrc;
|
||
|
|
unsigned long checksum = ~0U;
|
||
|
|
|
||
|
|
if (__get_word(ldq_u, first, src))
|
||
|
|
return 0;
|
||
|
|
lastsrc = 7+len+(unsigned long)src;
|
||
|
|
mskql(partial_dest, doff, partial_dest);
|
||
|
|
while (len >= 0) {
|
||
|
|
unsigned long second, word;
|
||
|
|
unsigned long second_dest;
|
||
|
|
|
||
|
|
if (__get_word(ldq_u, second, src+1))
|
||
|
|
return 0;
|
||
|
|
extql(first, soff, word);
|
||
|
|
checksum += carry;
|
||
|
|
len -= 8;
|
||
|
|
extqh(second, soff, first);
|
||
|
|
src++;
|
||
|
|
word |= first;
|
||
|
|
first = second;
|
||
|
|
insql(word, doff, second_dest);
|
||
|
|
checksum += word;
|
||
|
|
stq_u(partial_dest | second_dest, dst);
|
||
|
|
carry = checksum < word;
|
||
|
|
insqh(word, doff, partial_dest);
|
||
|
|
dst++;
|
||
|
|
}
|
||
|
|
len += doff;
|
||
|
|
checksum += carry;
|
||
|
|
if (len >= 0) {
|
||
|
|
unsigned long second, word;
|
||
|
|
unsigned long second_dest;
|
||
|
|
|
||
|
|
if (__get_word(ldq_u, second, lastsrc))
|
||
|
|
return 0;
|
||
|
|
extql(first, soff, word);
|
||
|
|
extqh(second, soff, first);
|
||
|
|
word |= first;
|
||
|
|
first = second;
|
||
|
|
mskql(word, len-doff, word);
|
||
|
|
checksum += word;
|
||
|
|
insql(word, doff, second_dest);
|
||
|
|
carry = checksum < word;
|
||
|
|
stq_u(partial_dest | second_dest, dst);
|
||
|
|
if (len) {
|
||
|
|
ldq_u(second_dest, dst+1);
|
||
|
|
insqh(word, doff, partial_dest);
|
||
|
|
mskqh(second_dest, len, second_dest);
|
||
|
|
stq_u(partial_dest | second_dest, dst+1);
|
||
|
|
}
|
||
|
|
checksum += carry;
|
||
|
|
} else {
|
||
|
|
unsigned long second, word;
|
||
|
|
unsigned long second_dest;
|
||
|
|
|
||
|
|
if (__get_word(ldq_u, second, lastsrc))
|
||
|
|
return 0;
|
||
|
|
extql(first, soff, word);
|
||
|
|
extqh(second, soff, first);
|
||
|
|
word |= first;
|
||
|
|
ldq_u(second_dest, dst);
|
||
|
|
mskql(word, len-doff, word);
|
||
|
|
checksum += word;
|
||
|
|
mskqh(second_dest, len, second_dest);
|
||
|
|
carry = checksum < word;
|
||
|
|
insql(word, doff, word);
|
||
|
|
stq_u(partial_dest | word | second_dest, dst);
|
||
|
|
checksum += carry;
|
||
|
|
}
|
||
|
|
return checksum;
|
||
|
|
}
|
||
|
|
|
||
|
|
static __wsum __csum_and_copy(const void __user *src, void *dst, int len)
|
||
|
|
{
|
||
|
|
unsigned long soff = 7 & (unsigned long) src;
|
||
|
|
unsigned long doff = 7 & (unsigned long) dst;
|
||
|
|
unsigned long checksum;
|
||
|
|
|
||
|
|
if (!doff) {
|
||
|
|
if (!soff)
|
||
|
|
checksum = csum_partial_cfu_aligned(
|
||
|
|
(const unsigned long __user *) src,
|
||
|
|
(unsigned long *) dst, len-8);
|
||
|
|
else
|
||
|
|
checksum = csum_partial_cfu_dest_aligned(
|
||
|
|
(const unsigned long __user *) src,
|
||
|
|
(unsigned long *) dst,
|
||
|
|
soff, len-8);
|
||
|
|
} else {
|
||
|
|
unsigned long partial_dest;
|
||
|
|
ldq_u(partial_dest, dst);
|
||
|
|
if (!soff)
|
||
|
|
checksum = csum_partial_cfu_src_aligned(
|
||
|
|
(const unsigned long __user *) src,
|
||
|
|
(unsigned long *) dst,
|
||
|
|
doff, len-8, partial_dest);
|
||
|
|
else
|
||
|
|
checksum = csum_partial_cfu_unaligned(
|
||
|
|
(const unsigned long __user *) src,
|
||
|
|
(unsigned long *) dst,
|
||
|
|
soff, doff, len-8, partial_dest);
|
||
|
|
}
|
||
|
|
return (__force __wsum)from64to16 (checksum);
|
||
|
|
}
|
||
|
|
|
||
|
|
__wsum
|
||
|
|
csum_and_copy_from_user(const void __user *src, void *dst, int len)
|
||
|
|
{
|
||
|
|
if (!access_ok(src, len))
|
||
|
|
return 0;
|
||
|
|
return __csum_and_copy(src, dst, len);
|
||
|
|
}
|
||
|
|
|
||
|
|
__wsum
|
||
|
|
csum_partial_copy_nocheck(const void *src, void *dst, int len)
|
||
|
|
{
|
||
|
|
return __csum_and_copy((__force const void __user *)src,
|
||
|
|
dst, len);
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL(csum_partial_copy_nocheck);
|