[llvm] specify NaN behavior more precisely (PR #66579)

[Nuno Lopes via llvm-commits]

nunoplopes wrote:

> I've a question regarding fpext & fptrunc: what's the payload we want there? We can't just copy the inputs.

Ok, I did the easy thing, which was to ignore the rules about copying the NaN payload from the input (in Alive2).
The good news is that there are only 4 tests failing. But they are related with fptrunc & fpext.

They are:
```llvm
; Transforms/InstCombine/maximum.ll
define float @reduce_precision(float %x, float %y) {
  %x.ext = fpext float %x to double
  %y.ext = fpext float %y to double
  %maximum = fmaximum double %x.ext, %y.ext
  %trunc = fptrunc double %maximum to float
  ret float %trunc
}
=>
define float @reduce_precision(float %x, float %y) {
  %maximum1 = fmaximum float %x, %y
  ret float %maximum1
}
Transformation doesn't verify!

ERROR: Value mismatch

Example:
float %x = #x00000000 (+0.0)
float %y = #x7f808000 (SNaN)

Source:
double %x.ext = #x0000000000000000 (+0.0)
double %y.ext = #x7ff8000000000000 (QNaN)
double %maximum = #x7ff8000000000000 (QNaN)
float %trunc = #x7fc00000 (QNaN)

Target:
float %maximum1 = #x7f808000 (SNaN)
Source value: #x7fc00000 (QNaN)
Target value: #x7f808000 (SNaN)
```

```llvm
; Transforms/InstCombine/select-binop-foldable-floating-point.ll
define float @select_fsub(i1 %cond, float %A, float %B) {
  %C = fsub float %A, %B
  %D = select i1 %cond, float %C, float %A
  ret float %D
}
=>
define float @select_fsub(i1 %cond, float %A, float %B) {
  %C = select i1 %cond, float %B, float 0.000000
  %D = fsub float %A, %C
  ret float %D
}
Transformation doesn't verify! (unsound)
ERROR: Target's return value is more undefined

Example:
i1 %cond = #x0 (0)
float %A = #x7f804000 (SNaN)
float %B = poison

Source:
float %C = poison
float %D = #x7f804000 (SNaN)

Target:
float %C = #x00000000 (+0.0)
float %D = #x7fd00000 (QNaN)
Source value: #x7f804000 (SNaN)
Target value: #x7fd00000 (QNaN)
```


```llvm
; Transforms/InstCombine/fpcast.ll
define half @test4-fast(float %a) {
  %b = fsub fast float -0.000000, %a
  %c = fptrunc float %b to half
  ret half %c
}
=>
define half @test4-fast(float %a) {
  %1 = fptrunc float %a to half
  %c = fneg fast half %1
  ret half %c
}
Transformation doesn't verify!

ERROR: Target is more poisonous than source

Example:
float %a = #xd8804000 (-1128098930098176)

Source:
float %b = #x7fc00000 (QNaN)
half %c = #x7e00 (QNaN)

Target:
half %1 = #xfc00 (-oo)
half %c = poison
Source value: #x7e00 (QNaN)
Target value: poison
```

```llvm
; Transforms/InstCombine/fpextend.ll
define float @test4(float %x) {
  %t1 = fpext float %x to double
  %t2 = fsub double -0.000000, %t1
  %t34 = fptrunc double %t2 to float
  ret float %t34
}
=>
define float @test4(float %x) {
  %t34 = fneg float %x
  ret float %t34
}
Transformation doesn't verify! (unsound)
ERROR: Value mismatch

Example:
float %x = #xff829ffe (SNaN)

Source:
double %t1 = #x7ff8000000000000 (QNaN)
double %t2 = #x7ff8000000000000 (QNaN)
float %t34 = #x7fc00000 (QNaN)

Target:
float %t34 = #x7f829ffe (SNaN)
Source value: #x7fc00000 (QNaN)
Target value: #x7f829ffe (SNaN)
```

https://github.com/llvm/llvm-project/pull/66579