Skip to content

Commit f06e5c1

Browse files
borsbors
committed
Auto merge of #139327 - cjgillot:gvn-place, r=oli-obk
Allow GVN to produce places and not just locals. That may be too big of a hammer, as we may introduce new deref projections (possible UB footgun + probably not good for perf). The second commit opts out of introducing projections that don't have a stable offset, which is probably what we want. Hence no new Deref and no new Index projections. Fixes #138936 cc `@scottmcm` `@dianqk`
2 parents 97c966b + d9caf84 commit f06e5c1

14 files changed

+206
-55
lines changed

‎compiler/rustc_mir_transform/src/gvn.rs

Lines changed: 47 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -980,27 +980,17 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
980980
}
981981
}
982982

983-
let tcx = self.tcx;
984-
let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new();
985-
loop {
986-
if let Some(local) = self.try_as_local(copy_from_local_value, location) {
987-
projection.reverse();
988-
let place = Place { local, projection: tcx.mk_place_elems(projection.as_slice()) };
989-
if rvalue.ty(self.local_decls, tcx) == place.ty(self.local_decls, tcx).ty {
990-
self.reused_locals.insert(local);
991-
*rvalue = Rvalue::Use(Operand::Copy(place));
992-
return Some(copy_from_value);
993-
}
994-
return None;
995-
} else if let Value::Projection(pointer, proj) = *self.get(copy_from_local_value)
996-
&& let Some(proj) = self.try_as_place_elem(proj, location)
997-
{
998-
projection.push(proj);
999-
copy_from_local_value = pointer;
1000-
} else {
1001-
return None;
983+
// Allow introducing places with non-constant offsets, as those are still better than
984+
// reconstructing an aggregate.
985+
if let Some(place) = self.try_as_place(copy_from_local_value, location, true) {
986+
if rvalue.ty(self.local_decls, self.tcx) == place.ty(self.local_decls, self.tcx).ty {
987+
self.reused_locals.insert(place.local);
988+
*rvalue = Rvalue::Use(Operand::Copy(place));
989+
return Some(copy_from_local_value);
1002990
}
1003991
}
992+
993+
None
1004994
}
1005995

1006996
fn simplify_aggregate(
@@ -1672,14 +1662,14 @@ fn op_to_prop_const<'tcx>(
16721662
}
16731663

16741664
impl<'tcx> VnState<'_, 'tcx> {
1675-
/// If either [`Self::try_as_constant`] as [`Self::try_as_local`] succeeds,
1665+
/// If either [`Self::try_as_constant`] as [`Self::try_as_place`] succeeds,
16761666
/// returns that result as an [`Operand`].
16771667
fn try_as_operand(&mut self, index: VnIndex, location: Location) -> Option<Operand<'tcx>> {
16781668
if let Some(const_) = self.try_as_constant(index) {
16791669
Some(Operand::Constant(Box::new(const_)))
1680-
} else if let Some(local) = self.try_as_local(index, location) {
1681-
self.reused_locals.insert(local);
1682-
Some(Operand::Copy(local.into()))
1670+
} else if let Some(place) = self.try_as_place(index, location, false) {
1671+
self.reused_locals.insert(place.local);
1672+
Some(Operand::Copy(place))
16831673
} else {
16841674
None
16851675
}
@@ -1712,6 +1702,35 @@ impl<'tcx> VnState<'_, 'tcx> {
17121702
Some(ConstOperand { span: DUMMY_SP, user_ty: None, const_ })
17131703
}
17141704

1705+
/// Construct a place which holds the same value as `index` and for which all locals strictly
1706+
/// dominate `loc`. If you used this place, add its base local to `reused_locals` to remove
1707+
/// storage statements.
1708+
#[instrument(level = "trace", skip(self), ret)]
1709+
fn try_as_place(
1710+
&mut self,
1711+
mut index: VnIndex,
1712+
loc: Location,
1713+
allow_complex_projection: bool,
1714+
) -> Option<Place<'tcx>> {
1715+
let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new();
1716+
loop {
1717+
if let Some(local) = self.try_as_local(index, loc) {
1718+
projection.reverse();
1719+
let place =
1720+
Place { local, projection: self.tcx.mk_place_elems(projection.as_slice()) };
1721+
return Some(place);
1722+
} else if let Value::Projection(pointer, proj) = *self.get(index)
1723+
&& (allow_complex_projection || proj.is_stable_offset())
1724+
&& let Some(proj) = self.try_as_place_elem(proj, loc)
1725+
{
1726+
projection.push(proj);
1727+
index = pointer;
1728+
} else {
1729+
return None;
1730+
}
1731+
}
1732+
}
1733+
17151734
/// If there is a local which is assigned `index`, and its assignment strictly dominates `loc`,
17161735
/// return it. If you used this local, add it to `reused_locals` to remove storage statements.
17171736
fn try_as_local(&mut self, index: VnIndex, loc: Location) -> Option<Local> {
@@ -1762,11 +1781,12 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
17621781
if let Some(value) = value {
17631782
if let Some(const_) = self.try_as_constant(value) {
17641783
*rvalue = Rvalue::Use(Operand::Constant(Box::new(const_)));
1765-
} else if let Some(local) = self.try_as_local(value, location)
1766-
&& *rvalue != Rvalue::Use(Operand::Move(local.into()))
1784+
} else if let Some(place) = self.try_as_place(value, location, false)
1785+
&& *rvalue != Rvalue::Use(Operand::Move(place))
1786+
&& *rvalue != Rvalue::Use(Operand::Copy(place))
17671787
{
1768-
*rvalue = Rvalue::Use(Operand::Copy(local.into()));
1769-
self.reused_locals.insert(local);
1788+
*rvalue = Rvalue::Use(Operand::Copy(place));
1789+
self.reused_locals.insert(place.local);
17701790
}
17711791
}
17721792
}

‎tests/mir-opt/const_prop/address_of_pair.fn0.GVN.diff

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,13 @@
3131
(*_3) = const true;
3232
_4 = const ();
3333
StorageDead(_4);
34-
- StorageLive(_5);
35-
+ nop;
34+
StorageLive(_5);
3635
StorageLive(_6);
3736
_6 = copy (_2.1: bool);
3837
_5 = Not(move _6);
3938
StorageDead(_6);
4039
_0 = copy _5;
41-
- StorageDead(_5);
42-
+ nop;
40+
StorageDead(_5);
4341
StorageDead(_3);
4442
StorageDead(_2);
4543
return;

‎tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,7 @@
6969

7070
bb0: {
7171
StorageLive(_1);
72-
- StorageLive(_2);
73-
+ nop;
72+
StorageLive(_2);
7473
StorageLive(_3);
7574
StorageLive(_11);
7675
StorageLive(_12);
@@ -119,8 +118,7 @@
119118
StorageDead(_11);
120119
_2 = &_3;
121120
_1 = copy _2;
122-
- StorageDead(_2);
123-
+ nop;
121+
StorageDead(_2);
124122
StorageLive(_4);
125123
- _9 = deref_copy _3;
126124
+ _9 = copy _3;
@@ -141,7 +139,7 @@
141139
StorageLive(_8);
142140
_8 = copy _5;
143141
- _7 = copy _8 as *mut () (PtrToPtr);
144-
+ _7 = copy _5 as *mut () (PtrToPtr);
142+
+ _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
145143
StorageDead(_8);
146144
StorageDead(_7);
147145
- StorageDead(_5);

‎tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,15 @@
3535

3636
bb0: {
3737
StorageLive(_1);
38-
- StorageLive(_2);
39-
+ nop;
38+
StorageLive(_2);
4039
StorageLive(_3);
4140
_3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue];
4241
}
4342

4443
bb1: {
4544
_2 = &_3;
4645
_1 = copy _2;
47-
- StorageDead(_2);
48-
+ nop;
46+
StorageDead(_2);
4947
StorageLive(_4);
5048
- _9 = deref_copy _3;
5149
+ _9 = copy _3;
@@ -66,7 +64,7 @@
6664
StorageLive(_8);
6765
_8 = copy _5;
6866
- _7 = copy _8 as *mut () (PtrToPtr);
69-
+ _7 = copy _5 as *mut () (PtrToPtr);
67+
+ _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
7068
StorageDead(_8);
7169
StorageDead(_7);
7270
- StorageDead(_5);

‎tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,7 @@
6969

7070
bb0: {
7171
StorageLive(_1);
72-
- StorageLive(_2);
73-
+ nop;
72+
StorageLive(_2);
7473
StorageLive(_3);
7574
StorageLive(_11);
7675
StorageLive(_12);
@@ -119,8 +118,7 @@
119118
StorageDead(_11);
120119
_2 = &_3;
121120
_1 = copy _2;
122-
- StorageDead(_2);
123-
+ nop;
121+
StorageDead(_2);
124122
StorageLive(_4);
125123
- _9 = deref_copy _3;
126124
+ _9 = copy _3;
@@ -141,7 +139,7 @@
141139
StorageLive(_8);
142140
_8 = copy _5;
143141
- _7 = copy _8 as *mut () (PtrToPtr);
144-
+ _7 = copy _5 as *mut () (PtrToPtr);
142+
+ _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
145143
StorageDead(_8);
146144
StorageDead(_7);
147145
- StorageDead(_5);

‎tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,15 @@
3535

3636
bb0: {
3737
StorageLive(_1);
38-
- StorageLive(_2);
39-
+ nop;
38+
StorageLive(_2);
4039
StorageLive(_3);
4140
_3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue];
4241
}
4342

4443
bb1: {
4544
_2 = &_3;
4645
_1 = copy _2;
47-
- StorageDead(_2);
48-
+ nop;
46+
StorageDead(_2);
4947
StorageLive(_4);
5048
- _9 = deref_copy _3;
5149
+ _9 = copy _3;
@@ -66,7 +64,7 @@
6664
StorageLive(_8);
6765
_8 = copy _5;
6866
- _7 = copy _8 as *mut () (PtrToPtr);
69-
+ _7 = copy _5 as *mut () (PtrToPtr);
67+
+ _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
7068
StorageDead(_8);
7169
StorageDead(_7);
7270
- StorageDead(_5);

‎tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-abort.diff

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
- // MIR for `compare_constant_index` before GVN
2+
+ // MIR for `compare_constant_index` after GVN
3+
4+
fn compare_constant_index(_1: [i32; 1], _2: [i32; 1]) -> std::cmp::Ordering {
5+
debug x => _1;
6+
debug y => _2;
7+
let mut _0: std::cmp::Ordering;
8+
let _3: &i32;
9+
let _4: usize;
10+
let mut _5: bool;
11+
let _6: &i32;
12+
let _7: usize;
13+
let mut _8: bool;
14+
scope 1 (inlined std::cmp::impls::<impl Ord for i32>::cmp) {
15+
let mut _9: i32;
16+
let mut _10: i32;
17+
}
18+
19+
bb0: {
20+
- StorageLive(_4);
21+
+ nop;
22+
_4 = const 0_usize;
23+
- _5 = Lt(copy _4, const 1_usize);
24+
- assert(move _5, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _4) -> [success: bb1, unwind unreachable];
25+
+ _5 = const true;
26+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb1, unwind unreachable];
27+
}
28+
29+
bb1: {
30+
- _3 = &_1[_4];
31+
+ _3 = &_1[0 of 1];
32+
StorageLive(_7);
33+
_7 = const 0_usize;
34+
- _8 = Lt(copy _7, const 1_usize);
35+
- assert(move _8, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _7) -> [success: bb2, unwind unreachable];
36+
+ _8 = const true;
37+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb2, unwind unreachable];
38+
}
39+
40+
bb2: {
41+
- _6 = &_2[_7];
42+
+ _6 = &_2[0 of 1];
43+
StorageLive(_9);
44+
- _9 = copy (*_3);
45+
+ _9 = copy _1[0 of 1];
46+
StorageLive(_10);
47+
- _10 = copy (*_6);
48+
+ _10 = copy _2[0 of 1];
49+
_0 = Cmp(move _9, move _10);
50+
StorageDead(_10);
51+
StorageDead(_9);
52+
StorageDead(_7);
53+
- StorageDead(_4);
54+
+ nop;
55+
return;
56+
}
57+
}
58+

‎tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-unwind.diff

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
- // MIR for `compare_constant_index` before GVN
2+
+ // MIR for `compare_constant_index` after GVN
3+
4+
fn compare_constant_index(_1: [i32; 1], _2: [i32; 1]) -> std::cmp::Ordering {
5+
debug x => _1;
6+
debug y => _2;
7+
let mut _0: std::cmp::Ordering;
8+
let _3: &i32;
9+
let _4: usize;
10+
let mut _5: bool;
11+
let _6: &i32;
12+
let _7: usize;
13+
let mut _8: bool;
14+
scope 1 (inlined std::cmp::impls::<impl Ord for i32>::cmp) {
15+
let mut _9: i32;
16+
let mut _10: i32;
17+
}
18+
19+
bb0: {
20+
- StorageLive(_4);
21+
+ nop;
22+
_4 = const 0_usize;
23+
- _5 = Lt(copy _4, const 1_usize);
24+
- assert(move _5, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _4) -> [success: bb1, unwind continue];
25+
+ _5 = const true;
26+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb1, unwind continue];
27+
}
28+
29+
bb1: {
30+
- _3 = &_1[_4];
31+
+ _3 = &_1[0 of 1];
32+
StorageLive(_7);
33+
_7 = const 0_usize;
34+
- _8 = Lt(copy _7, const 1_usize);
35+
- assert(move _8, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _7) -> [success: bb2, unwind continue];
36+
+ _8 = const true;
37+
+ assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb2, unwind continue];
38+
}
39+
40+
bb2: {
41+
- _6 = &_2[_7];
42+
+ _6 = &_2[0 of 1];
43+
StorageLive(_9);
44+
- _9 = copy (*_3);
45+
+ _9 = copy _1[0 of 1];
46+
StorageLive(_10);
47+
- _10 = copy (*_6);
48+
+ _10 = copy _2[0 of 1];
49+
_0 = Cmp(move _9, move _10);
50+
StorageDead(_10);
51+
StorageDead(_9);
52+
StorageDead(_7);
53+
- StorageDead(_4);
54+
+ nop;
55+
return;
56+
}
57+
}
58+

‎tests/mir-opt/gvn_copy_constant_projection.rs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
2+
3+
use std::cmp::Ordering;
4+
fn compare_constant_index(x: [i32; 1], y: [i32; 1]) -> Ordering {
5+
// CHECK-LABEL: fn compare_constant_index(
6+
// CHECK-NOT: (*{{_.*}});
7+
// CHECK: [[lhs:_.*]] = copy _1[0 of 1];
8+
// CHECK-NOT: (*{{_.*}});
9+
// CHECK: [[rhs:_.*]] = copy _2[0 of 1];
10+
// CHECK: _0 = Cmp(move [[lhs]], move [[rhs]]);
11+
Ord::cmp(&x[0], &y[0])
12+
}
13+
14+
fn main() {
15+
compare_constant_index([1], [2]);
16+
}
17+
18+
// EMIT_MIR gvn_copy_constant_projection.compare_constant_index.GVN.diff

0 commit comments

Comments
 (0)