Auto merge of #125829 - petrochenkov:upctxt2, r= · rust-lang/rust@225796f (original) (raw)

`@@ -4,10 +4,10 @@ use crate::SPAN_TRACK;

4

`use crate::{BytePos, SpanData};

5

6

`use rustc_data_structures::fx::FxIndexSet;

7

-

8

7

`// This code is very hot and uses lots of arithmetic, avoid overflow checks for performance.

9

8

`// See https://github.com/rust-lang/rust/pull/119440#issuecomment-1874255727

10

9

`use rustc_serialize::int_overflow::DebugStrictAdd;

10

use std::mem::transmute;

11

12

`/// A compressed span.

13

`///

`@@ -87,6 +87,115 @@ pub struct Span {

87

`ctxt_or_parent_or_marker: u16,

88

89

90

// Convenience structures for all span formats.

91

#[derive(Clone, Copy)]

92

struct InlineCtxt {

93

lo: u32,

94

len: u16,

95

ctxt: u16,

96

}

97

+

98

#[derive(Clone, Copy)]

99

struct InlineParent {

100

lo: u32,

101

len_with_tag: u16,

102

parent: u16,

103

}

104

+

105

#[derive(Clone, Copy)]

106

struct PartiallyInterned {

107

index: u32,

108

_marker1: u16,

109

ctxt: u16,

110

}

111

+

112

#[derive(Clone, Copy)]

113

struct Interned {

114

index: u32,

115

_marker1: u16,

116

_marker2: u16,

117

}

118

+

119

enum Fmt<'a> {

120

InlineCtxt(&'a mut InlineCtxt),

121

InlineParent(&'a mut InlineParent),

122

PartiallyInterned(&'a mut PartiallyInterned),

123

Interned(&'a mut Interned),

124

}

125

+

126

impl InlineCtxt {

127

#[inline]

128

fn data(self) -> SpanData {

129

let len = self.len as u32;

130

debug_assert!(len <= MAX_LEN);

131

SpanData {

132

lo: BytePos(self.lo),

133

hi: BytePos(self.lo.debug_strict_add(len)),

134

ctxt: SyntaxContext::from_u32(self.ctxt as u32),

135

parent: None,

136

}

137

}

138

#[inline]

139

fn to_span(self) -> Span {

140

unsafe { transmute(self) }

141

}

142

}

143

+

144

impl InlineParent {

145

#[inline]

146

fn data(self) -> SpanData {

147

let len = (self.len_with_tag & !PARENT_TAG) as u32;

148

debug_assert!(len <= MAX_LEN);

149

SpanData {

150

lo: BytePos(self.lo),

151

hi: BytePos(self.lo.debug_strict_add(len)),

152

ctxt: SyntaxContext::root(),

153

parent: Some(LocalDefId { local_def_index: DefIndex::from_u32(self.parent as u32) }),

154

}

155

}

156

#[inline]

157

fn to_span(self) -> Span {

158

unsafe { transmute(self) }

159

}

160

}

161

+

162

impl PartiallyInterned {

163

#[inline]

164

fn data(self) -> SpanData {

165

SpanData {

166

ctxt: SyntaxContext::from_u32(self.ctxt as u32),

167

..with_span_interner(|interner| interner.spans[self.index as usize])

168

}

169

}

170

#[inline]

171

fn to_span(self) -> Span {

172

unsafe { transmute(self) }

173

}

174

}

175

+

176

impl Interned {

177

#[inline]

178

fn data(self) -> SpanData {

179

with_span_interner(|interner| interner.spans[self.index as usize])

180

}

181

#[inline]

182

fn to_span(self) -> Span {

183

unsafe { transmute(self) }

184

}

185

}

186

+

187

impl Fmt<'_> {

188

#[inline]

189

fn data(self) -> SpanData {

190

match self {

191

Fmt::InlineCtxt(span) => span.data(),

192

Fmt::InlineParent(span) => span.data(),

193

Fmt::PartiallyInterned(span) => span.data(),

194

Fmt::Interned(span) => span.data(),

195

}

196

}

197

}

198

+

90

199

`` // MAX_LEN is chosen so that PARENT_TAG | MAX_LEN is distinct from

91

200

`` // BASE_LEN_INTERNED_MARKER. (If MAX_LEN was 1 higher, this wouldn't be true.)

92

201

`const MAX_LEN: u32 = 0b0111_1111_1111_1110;

`@@ -111,42 +220,48 @@ impl Span {

111

220

` std::mem::swap(&mut lo, &mut hi);

112

221

113

222

114

let (lo2, len, ctxt2) = (lo.0, hi.0 - lo.0, ctxt.as_u32());

115

-

223

// Small len may enable one of fully inline formats (or may not).

224

let (len, ctxt32) = (hi.0 - lo.0, ctxt.as_u32());

116

225

`if len <= MAX_LEN {

117

if ctxt2 <= MAX_CTXT && parent.is_none() {

118

// Inline-context format.

119

return Span {

120

lo_or_index: lo2,

121

len_with_tag_or_marker: len as u16,

122

ctxt_or_parent_or_marker: ctxt2 as u16,

123

};

124

} else if ctxt2 == SyntaxContext::root().as_u32()

226

if ctxt32 <= MAX_CTXT && parent.is_none() {

227

return InlineCtxt { lo: lo.0, len: len as u16, ctxt: ctxt32 as u16 }.to_span();

228

} else if ctxt32 == 0

125

229

` && let Some(parent) = parent

126

&& let parent2 = parent.local_def_index.as_u32()

127

&& parent2 <= MAX_CTXT

230

&& let parent32 = parent.local_def_index.as_u32()

231

&& parent32 <= MAX_CTXT

128

232

129

// Inline-parent format.

130

return Span {

131

lo_or_index: lo2,

132

len_with_tag_or_marker: PARENT_TAG | len as u16,

133

ctxt_or_parent_or_marker: parent2 as u16,

134

};

233

let len_with_tag = PARENT_TAG | len as u16;

234

return InlineParent { lo: lo.0, len_with_tag, parent: parent32 as u16 }.to_span();

135

235

136

236

137

237

138

// Partially-interned or fully-interned format.

139

let index =

140

with_span_interner(|interner| interner.intern(&SpanData { lo, hi, ctxt, parent }));

141

let ctxt_or_parent_or_marker = if ctxt2 <= MAX_CTXT {

142

ctxt2 as u16 // partially-interned

143

} else {

144

CTXT_INTERNED_MARKER // fully-interned

238

// Otherwise small ctxt may enable the partially inline format.

239

let index = |ctxt| {

240

with_span_interner(|interner| interner.intern(&SpanData { lo, hi, ctxt, parent }))

145

241

`};

146

Span {

147

lo_or_index: index,

148

len_with_tag_or_marker: BASE_LEN_INTERNED_MARKER,

149

ctxt_or_parent_or_marker,

242

if ctxt32 <= MAX_CTXT {

243

let index = index(SyntaxContext::from_u32(u32::MAX)); // any value, should never be read

244

PartiallyInterned { index, _marker1: BASE_LEN_INTERNED_MARKER, ctxt: ctxt32 as u16 }

245

.to_span()

246

} else {

247

let index = index(ctxt);

248

Interned { index, _marker1: BASE_LEN_INTERNED_MARKER, _marker2: CTXT_INTERNED_MARKER }

249

.to_span()

250

}

251

}

252

+

253

#[inline]

254

fn fmt(&mut self) -> Fmt<'_> {

255

if self.len_with_tag_or_marker != BASE_LEN_INTERNED_MARKER {

256

if self.len_with_tag_or_marker & PARENT_TAG == 0 {

257

Fmt::InlineCtxt(unsafe { transmute(self) })

258

} else {

259

Fmt::InlineParent(unsafe { transmute(self) })

260

}

261

} else if self.ctxt_or_parent_or_marker != CTXT_INTERNED_MARKER {

262

Fmt::PartiallyInterned(unsafe { transmute(self) })

263

} else {

264

Fmt::Interned(unsafe { transmute(self) })

150

265

151

266

152

267

`@@ -162,39 +277,8 @@ impl Span {

162

277

`/// Internal function to translate between an encoded span and the expanded representation.

163

278

`/// This function must not be used outside the incremental engine.

164

279

`#[inline]

165

pub fn data_untracked(self) -> SpanData {

166

if self.len_with_tag_or_marker != BASE_LEN_INTERNED_MARKER {

167

if self.len_with_tag_or_marker & PARENT_TAG == 0 {

168

// Inline-context format.

169

let len = self.len_with_tag_or_marker as u32;

170

debug_assert!(len <= MAX_LEN);

171

SpanData {

172

lo: BytePos(self.lo_or_index),

173

hi: BytePos(self.lo_or_index.debug_strict_add(len)),

174

ctxt: SyntaxContext::from_u32(self.ctxt_or_parent_or_marker as u32),

175

parent: None,

176

}

177

} else {

178

// Inline-parent format.

179

let len = (self.len_with_tag_or_marker & !PARENT_TAG) as u32;

180

debug_assert!(len <= MAX_LEN);

181

let parent = LocalDefId {

182

local_def_index: DefIndex::from_u32(self.ctxt_or_parent_or_marker as u32),

183

};

184

SpanData {

185

lo: BytePos(self.lo_or_index),

186

hi: BytePos(self.lo_or_index.debug_strict_add(len)),

187

ctxt: SyntaxContext::root(),

188

parent: Some(parent),

189

}

190

}

191

} else {

192

// Fully-interned or partially-interned format. In either case,

193

// the interned value contains all the data, so we don't need to

194

// distinguish them.

195

let index = self.lo_or_index;

196

with_span_interner(|interner| interner.spans[index as usize])

197

}

280

pub fn data_untracked(mut self) -> SpanData {

281

self.fmt().data()

198

282

199

283

200

284

`` /// Returns true if this is a dummy span with any hygienic context.

`@@ -214,26 +298,28 @@ impl Span {

214

298

215

299

216

300

301

// For optimization we are interested in cases in which the context is inline and the context

302

// update doesn't change format. All non-inline or format changing scenarios require accessing

303

`` +

// interner and can fall back to Span::new.

304

#[inline]

305

pub fn update_ctxt(&mut self, update: impl FnOnce(SyntaxContext) -> SyntaxContext) {

306

// FIXME(#125017): Update ctxt inline without touching interner when possible.

307

let data = self.data();

308

*self = data.with_ctxt(update(data.ctxt));

309

}

310

+

217

311

`// Returns either syntactic context, if it can be retrieved without taking the interner lock,

218

312

`// or an index into the interner if it cannot.

219

fn inline_ctxt(self) -> Result<SyntaxContext, usize> {

220

Ok(if self.len_with_tag_or_marker != BASE_LEN_INTERNED_MARKER {

221

if self.len_with_tag_or_marker & PARENT_TAG == 0 {

222

// Inline-context format.

223

SyntaxContext::from_u32(self.ctxt_or_parent_or_marker as u32)

224

} else {

225

// Inline-parent format. We know that the SyntaxContext is root.

226

SyntaxContext::root()

313

#[inline]

314

fn inline_ctxt(mut self) -> Result<SyntaxContext, usize> {

315

match self.fmt() {

316

Fmt::InlineCtxt(InlineCtxt { ctxt, .. })

317

| Fmt::PartiallyInterned(PartiallyInterned { ctxt, .. }) => {

318

Ok(SyntaxContext::from_u32(*ctxt as u32))

227

319

228

} else if self.ctxt_or_parent_or_marker != CTXT_INTERNED_MARKER {

229

// Partially-interned format. This path avoids looking up the

230

// interned value, and is the whole point of the

231

// partially-interned format.

232

SyntaxContext::from_u32(self.ctxt_or_parent_or_marker as u32)

233

} else {

234

// Fully-interned format.

235

return Err(self.lo_or_index as usize);

236

})

320

Fmt::InlineParent(_) => Ok(SyntaxContext::root()),

321

Fmt::Interned(span) => Err(span.index as usize),

322

}

237

323

238

324

239

325

`` /// This function is used as a fast path when decoding the full SpanData is not necessary.