Coverage for src/indium/segments.py: 99%

1"""Grapheme-aware text operations without regex library dependency.

2

3 This module provides safe text slicing and truncation that respects grapheme

4 cluster boundaries. Prevents breaking emoji sequences, combining character

5 sequences, and other multi-codepoint visual units.

6

7 Implementation: UAX #29 (Unicode Text Segmentation)

8"""

9

10 import bisect

11 from collections.abc import Iterator

12 from functools import lru_cache

13 from typing import Optional

14

15 from ._grapheme_data import (

16 CONTROL,

17 CR,

18 EXTEND,

19 EXTENDED_PICTOGRAPHIC,

20 GRAPHEME_BREAK_RANGES,

21 INCB_CONSONANT,

22 INCB_EXTEND,

23 INCB_LINKER,

24 LF,

25 LV,

26 LVT,

27 PREPEND,

28 REGIONAL_INDICATOR,

29 SPACINGMARK,

30 ZWJ,

31 L,

32 T,

33 V,

34)

35

36

37@lru_cache(maxsize=4096)

38 def _get_break_property(codepoint: int) -> int:

39 """Get Grapheme_Cluster_Break property for a code point."""

40 # Binary search in the RLE table

41 # index-1 gives the range containing codepoint (or starting before it)

42 idx = bisect.bisect_right(GRAPHEME_BREAK_RANGES, (codepoint, 999))

43 if idx == 0:

44 return 0 # Should not happen if table covers 0

45

46 start, prop = GRAPHEME_BREAK_RANGES[idx - 1]

47 return prop

48

49

50 def safe_truncate(text: str, max_graphemes: int) -> str:

51 """Truncate text to max grapheme clusters, not code points.

52

53 Ensures cut doesn't break:

54 - Emoji sequences (ZWJ, skin tone modifiers)

55 - Combining character sequences (base + marks)

56 - Regional indicator pairs (flag emoji)

57

58 Args:

59 text: Input string

60 max_graphemes: Maximum number of visual units (grapheme clusters)

61

62 Returns:

63 Truncated string at valid grapheme boundary

64

65 Raises:

66 ValueError: If max_graphemes is negative

67

68 Examples:

69 >>> safe_truncate("hello", 3)

70 'hel'

71 >>> safe_truncate("café", 3) # é is one grapheme

72 'caf'

73 >>> safe_truncate("👨‍👩‍👧", 1) # Family emoji is one grapheme

74 '👨\u200d👩\u200d👧'

75 >>> safe_truncate("hello👋🏽world", 6) # Waving hand with skin tone

76 'hello👋🏽'

77 """

78 if max_graphemes < 0:

79 raise ValueError(f"max_graphemes must be non-negative, got {max_graphemes}")

80

81 if max_graphemes == 0:

82 return ""

83

84 grapheme_count = 0

85 pos = 0

86

87 while pos < len(text) and grapheme_count < max_graphemes:

88 # Find end of current grapheme

89 grapheme_end = _find_grapheme_end(text, pos)

90 pos = grapheme_end

91 grapheme_count += 1

92

93 return text[:pos]

94

95

96 def count_graphemes(text: str) -> int:

97 """Count grapheme clusters (visual units) in text.

98

99 Args:

100 text: Input string

101

102 Returns:

103 Number of grapheme clusters

104

105 Examples:

106 >>> count_graphemes("hello")

107 5

108 >>> count_graphemes("café") # é = e + combining acute

109 4

110 >>> count_graphemes("👨‍👩‍👧") # Family emoji

111 1

112 >>> count_graphemes("hello👋🏽") # Waving with skin tone

113 6

114 """

115 count = 0

116 pos = 0

117

118 while pos < len(text):

119 grapheme_end = _find_grapheme_end(text, pos)

120 count += 1

121 pos = grapheme_end

122

123 return count

124

125

126 def grapheme_slice(text: str, start: int, end: Optional[int] = None) -> str:

127 """Slice text by grapheme indices, not code points.

128

129 Args:

130 text: Input string

131 start: Start grapheme index (inclusive)

132 end: End grapheme index (exclusive). If None, slice to end

133

134 Returns:

135 Substring from start to end grapheme indices

136

137 Raises:

138 ValueError: If start is negative or end < start

139

140 Examples:

141 >>> grapheme_slice("hello", 1, 4)

142 'ell'

143 >>> grapheme_slice("café", 0, 3)

144 'caf'

145 >>> grapheme_slice("👨‍👩‍👧test", 1, 3)

146 'te'

147 """

148 if start < 0:

149 raise ValueError(f"start must be non-negative, got {start}")

150

151 if end is not None and end < start:

152 raise ValueError(f"end ({end}) must be >= start ({start})")

153

154 # Find start position in code points

155 grapheme_count = 0

156 start_pos = 0

157

158 while start_pos < len(text) and grapheme_count < start:

159 grapheme_end = _find_grapheme_end(text, start_pos)

160 start_pos = grapheme_end

161 grapheme_count += 1

162

163 # If we ran out of text before reaching start, return empty

164 if start_pos >= len(text):

165 return ""

166

167 # If no end specified, return from start to end of text

168 if end is None:

169 return text[start_pos:]

170

171 # Find end position in code points

172 end_pos = start_pos

173 while end_pos < len(text) and grapheme_count < end:

174 grapheme_end = _find_grapheme_end(text, end_pos)

175 end_pos = grapheme_end

176 grapheme_count += 1

177

178 return text[start_pos: end_pos]

179

180

181 def iter_graphemes(text: str) -> Iterator[str]:

182 """Iterate over grapheme clusters.

183

184 Args:

185 text: Input string

186

187 Yields:

188 Individual grapheme clusters

189

190 Examples:

191 >>> list(iter_graphemes("hello"))

192 ['h', 'e', 'l', 'l', 'o']

193 >>> list(iter_graphemes("café"))

194 ['c', 'a', 'f', 'é']

195 >>> list(iter_graphemes("a👋🏽b"))

196 ['a', '👋🏽', 'b']

197 """

198 pos = 0

199

200 while pos < len(text):

201 grapheme_end = _find_grapheme_end(text, pos)

202 yield text[pos: grapheme_end]

203 pos = grapheme_end

204

205

206# Internal helper: Find end of grapheme cluster starting at pos

207 def _find_grapheme_end(text: str, pos: int) -> int:

208 """Find the end position of the grapheme cluster starting at pos.

209

210 Args:

211 text: Input string

212 pos: Start position of grapheme

213

214 Returns:

215 Position after the end of the grapheme cluster

216 """

217 if pos >= len(text):

218 return pos

219

220 # Start with first character

221 end = pos + 1

222

223 # Continue while we're not at a grapheme boundary

224 while end < len(text) and not _is_grapheme_boundary(text, end):

225 end += 1

226

227 return end

228

229

230 def _is_grapheme_boundary(text: str, pos: int) -> bool:

231 """Check if position is a valid grapheme boundary.

232

233 Implements Unicode TR29 (Grapheme Cluster Boundaries).

234

235 Args:

236 text: Input string

237 pos: Position to check

238

239 Returns:

240 True if position is a valid grapheme boundary

241 """

242 if pos == 0 or pos >= len(text):

243 return True

244

245 curr_char = text[pos]

246 prev_char = text[pos - 1]

247

248 curr_prop = _get_break_property(ord(curr_char))

249 prev_prop = _get_break_property(ord(prev_char))

250

251 # GB3: CR x LF

252 if prev_prop == CR and curr_prop == LF:

253 return False

254

255 # GB4: (Control | CR | LF) ÷

256 if prev_prop in (CONTROL, CR, LF):

257 return True

258

259 # GB5: ÷ (Control | CR | LF)

260 if curr_prop in (CONTROL, CR, LF):

261 return True

262

263 # GB6: L x (L | V | LV | LVT)

264 if prev_prop == L and curr_prop in (L, V, LV, LVT):

265 return False

266

267 # GB7: (LV | V) x (V | T)

268 if prev_prop in (LV, V) and curr_prop in (V, T):

269 return False

270

271 # GB8: (LVT | T) x T

272 if prev_prop in (LVT, T) and curr_prop == T:

273 return False

274

275 # GB9: x (Extend | ZWJ)

276 # Note: InCB_Linker and InCB_Extend must also be treated as Extend for GB9

277 if curr_prop in (EXTEND, ZWJ, INCB_EXTEND, INCB_LINKER):

278 return False

279

280 # GB9a: x SpacingMark

281 if curr_prop == SPACINGMARK:

282 return False

283

284 # GB9b: Prepend x

285 if prev_prop == PREPEND:

286 return False

287

288 # GB9c: \p{InCB=Linker} [ \p{InCB=Extend} \p{InCB=Linker} \p{Zwj} ]* x \p{InCB=Consonant}

289 # Restriction: The Linker itself must be part of a valid syllable, i.e., preceded by Consonant.

290 # This fixes failures where 'a' + Virama + 'Ta' breaks (because 'a' is not InCB Consonant).

291 if curr_prop == INCB_CONSONANT:

292 # 1. Scan backwards for Linker

293 i = pos - 1

294 found_linker = False

295 linker_index = -1

296

297 while i >= 0:

298 prop = _get_break_property(ord(text[i]))

299 if prop == INCB_LINKER:

300 found_linker = True

301 linker_index = i

302 break

303 if prop not in (EXTEND, ZWJ, INCB_EXTEND, INCB_LINKER):

304 break # Sequence broken before finding Linker

305 i -= 1

306

307 if found_linker:

308 # 2. Verify Linker is attached to a Consonant

309 # Scan backwards from Linker skipping Extend/ZWJ/Linker

310 j = linker_index - 1

311 valid_base = False

312 while j >= 0:

313 prop_j = _get_break_property(ord(text[j]))

314 if prop_j == INCB_CONSONANT:

315 valid_base = True

316 break

317 if prop_j not in (EXTEND, ZWJ, INCB_EXTEND, INCB_LINKER):

318 break # Hit start of cluster or invalid char

319 j -= 1

320

321 if valid_base:

322 return False

323

324 # GB11: \p{Extended_Pictographic} Extend* ZWJ x \p{Extended_Pictographic}

325 if prev_prop == ZWJ and curr_prop == EXTENDED_PICTOGRAPHIC:

326 # Scan backwards to see if ZWJ is preceded by Extended_Pictographic + Extend*

327 i = pos - 2

328 while i >= 0:

329 prop = _get_break_property(ord(text[i]))

330 if prop == EXTENDED_PICTOGRAPHIC:

331 return False

332 # Treat InCB properties as Extend for this rule too?

333 # UAX #29 says "Extend". InCB_Extend and InCB_Linker ARE Extend.

334 if prop not in (EXTEND, INCB_EXTEND, INCB_LINKER):

335 break

336 i -= 1

337

338 # GB12/GB13: Regional_Indicator sequence

339 if prev_prop == REGIONAL_INDICATOR and curr_prop == REGIONAL_INDICATOR:

340 # We need to count RI characters backwards from pos

341 # If count is odd, it's a valid pair (break after second).

342 # If count is even, we are in middle of new pair (no break).

343 # The rule effectively says: "Do not break within a pair".

344 # A pair starts at an even offset from the start of the RI sequence.

345

346 ri_count = 0

347 i = pos - 1

348 while i >= 0 and _get_break_property(ord(text[i])) == REGIONAL_INDICATOR:

349 ri_count += 1

350 i -= 1

351

352 # If we have seen an odd number of RIs before 'curr',

353 # then 'prev' and 'curr' form a pair.

354 if ri_count % 2 == 1:

355 return False

356

357 # GB999: Any ÷ Any

358 return True