Cleanup for cryptographic algorithms vs. compiler optimizations

[Dag-Erling Smørgrav]

Tijl Coosemans <tijl at coosemans.org> writes:
> Dag-Erling Smørgrav <des at des.no> writes:
> > #define FORCE_ASSIGN(type, var, value) \
> >         *(volatile type *)&(var) = (value)
> memset can be optimised away as well. The only way is to declare those
> variables volatile.

Assigning through a volatile pointer, as in FORCE_ASSIGN(), also works,
even if the variable itself is not volatile.  Unfortunately, you can't
trick the compiler into not optimizing away memset(), since you can't
pass a volatile pointer to memset().

% cat >zero.c <<EOF
#include <string.h>

int
main(void)
{
	char a, b, c;

	a = 1;
	memset(&b, 2, sizeof b);
	*(volatile char *)&c = 3;
	return 0;
}
EOF
% for O in 0 1 2 ; do c99 -O$O -o zero-O$O zero.c ; done
% for O in 0 1 2 ; do echo disassemble main | gdb -batch -x /dev/stdin -q zero-O$O ; done
Dump of assembler code for function main:
0x0000000000400560 <main+0>:	push   %rbp
0x0000000000400561 <main+1>:	mov    %rsp,%rbp
0x0000000000400564 <main+4>:	sub    $0x10,%rsp
0x0000000000400568 <main+8>:	movb   $0x1,0xffffffffffffffff(%rbp)
0x000000000040056c <main+12>:	lea    0xfffffffffffffffe(%rbp),%rdi
0x0000000000400570 <main+16>:	mov    $0x1,%edx
0x0000000000400575 <main+21>:	mov    $0x2,%esi
0x000000000040057a <main+26>:	callq  0x40042c <memset>
0x000000000040057f <main+31>:	lea    0xfffffffffffffffd(%rbp),%rax
0x0000000000400583 <main+35>:	movb   $0x3,(%rax)
0x0000000000400586 <main+38>:	mov    $0x0,%eax
0x000000000040058b <main+43>:	leaveq 
0x000000000040058c <main+44>:	retq   
0x000000000040058d <main+45>:	nop    
0x000000000040058e <main+46>:	nop    
0x000000000040058f <main+47>:	nop    
End of assembler dump.
Dump of assembler code for function main:
0x0000000000400520 <main+0>:	movb   $0x3,0xffffffffffffffff(%rsp)
0x0000000000400525 <main+5>:	mov    $0x0,%eax
0x000000000040052a <main+10>:	retq   
0x000000000040052b <main+11>:	nop    
End of assembler dump.
Dump of assembler code for function main:
0x0000000000400520 <main+0>:	xor    %eax,%eax
0x0000000000400522 <main+2>:	movb   $0x3,0xffffffffffffffff(%rsp)
0x0000000000400527 <main+7>:	retq   
End of assembler dump.

In the -O0 case, all three assignments are carried out.  In the -O1 and
-O2 cases, the first two assignments and the corresponding variables are
optimized away, while the third (which uses the volatile pointer trick)
is not.

Clang produces significantly worse code than gcc in all cases:

% for O in 0 1 2 ; do clang -O$O -o zero-O$O zero.c ; done
% for O in 0 1 2 ; do echo disassemble main | gdb -batch -x /dev/stdin -q zero-O$O ; done
Dump of assembler code for function main:
0x0000000000400550 <main+0>:	push   %rbp
0x0000000000400551 <main+1>:	mov    %rsp,%rbp
0x0000000000400554 <main+4>:	movl   $0x0,0xfffffffffffffffc(%rbp)
0x000000000040055b <main+11>:	movb   $0x1,0xfffffffffffffffb(%rbp)
0x000000000040055f <main+15>:	movb   $0x2,0xfffffffffffffffa(%rbp)
0x0000000000400563 <main+19>:	movb   $0x3,0xfffffffffffffff9(%rbp)
0x0000000000400567 <main+23>:	movl   $0x0,0xfffffffffffffffc(%rbp)
0x000000000040056e <main+30>:	mov    0xfffffffffffffffc(%rbp),%eax
0x0000000000400571 <main+33>:	pop    %rbp
0x0000000000400572 <main+34>:	retq   
0x0000000000400573 <main+35>:	nop    
End of assembler dump.
Dump of assembler code for function main:
0x0000000000400550 <main+0>:	push   %rbp
0x0000000000400551 <main+1>:	mov    %rsp,%rbp
0x0000000000400554 <main+4>:	movb   $0x3,0xffffffffffffffff(%rbp)
0x0000000000400558 <main+8>:	xor    %eax,%eax
0x000000000040055a <main+10>:	pop    %rbp
0x000000000040055b <main+11>:	retq   
End of assembler dump.
Dump of assembler code for function main:
0x0000000000400550 <main+0>:	push   %rbp
0x0000000000400551 <main+1>:	mov    %rsp,%rbp
0x0000000000400554 <main+4>:	movb   $0x3,0xffffffffffffffff(%rbp)
0x0000000000400558 <main+8>:	xor    %eax,%eax
0x000000000040055a <main+10>:	pop    %rbp
0x000000000040055b <main+11>:	retq   
End of assembler dump.

DES
-- 
Dag-Erling Smørgrav - des at des.no