1 | """Glob_.py."""
|
2 |
|
3 | import libc
|
4 |
|
5 | from _devbuild.gen.id_kind_asdl import Id, Id_t
|
6 | from _devbuild.gen.syntax_asdl import (
|
7 | CompoundWord,
|
8 | Token,
|
9 | word_part_e,
|
10 | glob_part,
|
11 | glob_part_e,
|
12 | glob_part_t,
|
13 | )
|
14 | from core import pyutil
|
15 | from frontend import match
|
16 | from mycpp import mylib
|
17 | from mycpp.mylib import log, print_stderr
|
18 |
|
19 | from typing import List, Tuple, cast, TYPE_CHECKING
|
20 | if TYPE_CHECKING:
|
21 | from core import optview
|
22 | from frontend.match import SimpleLexer
|
23 |
|
24 | _ = log
|
25 |
|
26 |
|
27 | def LooksLikeGlob(s):
|
28 | # type: (str) -> bool
|
29 | """Does this string look like a glob pattern?
|
30 |
|
31 | Like other shells, OSH avoids calls to glob() unless there are glob
|
32 | metacharacters.
|
33 |
|
34 | TODO: Reference lib/glob / glob_pattern functions in bash
|
35 | $ grep glob_pattern lib/glob/*
|
36 |
|
37 | Used:
|
38 | 1. in Globber below
|
39 | 2. for the slow path / fast path of prefix/suffix/patsub ops.
|
40 | """
|
41 | left_bracket = False
|
42 | i = 0
|
43 | n = len(s)
|
44 | while i < n:
|
45 | c = mylib.ByteAt(s, i)
|
46 |
|
47 | if mylib.ByteEquals(c, '\\'):
|
48 | i += 1
|
49 |
|
50 | elif mylib.ByteEquals(c, '*') or mylib.ByteEquals(c, '?'):
|
51 | return True
|
52 |
|
53 | elif mylib.ByteEquals(c, '['):
|
54 | left_bracket = True
|
55 |
|
56 | elif mylib.ByteEquals(c, ']') and left_bracket:
|
57 | # It has at least one pair of balanced []. Not bothering to check stray
|
58 | # [ or ].
|
59 | return True
|
60 |
|
61 | i += 1
|
62 | return False
|
63 |
|
64 |
|
65 | def LooksLikeStaticGlob(w):
|
66 | # type: (CompoundWord) -> bool
|
67 | """Like LooksLikeGlob, but for static words."""
|
68 |
|
69 | left_bracket = False
|
70 | for part in w.parts:
|
71 | if part.tag() == word_part_e.Literal:
|
72 | id_ = cast(Token, part).id
|
73 | if id_ in (Id.Lit_Star, Id.Lit_QMark):
|
74 | return True
|
75 | elif id_ == Id.Lit_LBracket:
|
76 | left_bracket = True
|
77 | elif id_ == Id.Lit_RBracket and left_bracket:
|
78 | return True
|
79 | return False
|
80 |
|
81 |
|
82 | # Glob Helpers for WordParts.
|
83 | # NOTE: Escaping / doesn't work, because it's not a filename character.
|
84 | # ! : - are metachars within character classes
|
85 | # ( ) | are extended glob characters, and it's OK to add extra \ when the
|
86 | # underlying library doesn't support extended globs
|
87 | # we don't need to escape the @ in @(cc), because escaping ( is enough
|
88 | GLOB_META_CHARS = r'\*?[]-:!()|'
|
89 |
|
90 |
|
91 | def GlobEscape(s):
|
92 | # type: (str) -> str
|
93 | """For SingleQuoted, DoubleQuoted, and EscapedLiteral."""
|
94 | return pyutil.BackslashEscape(s, GLOB_META_CHARS)
|
95 |
|
96 |
|
97 | # Bug fix: add [] so [[:space:]] is not special, etc.
|
98 | ERE_META_CHARS = r'\?*+{}^$.()|[]'
|
99 |
|
100 |
|
101 | def ExtendedRegexEscape(s):
|
102 | # type: (str) -> str
|
103 | """Quoted parts need to be regex-escaped when quoted, e.g. [[ $a =~ "{" ]].
|
104 | I don't think libc has a function to do this. Escape these characters:
|
105 |
|
106 | https://www.gnu.org/software/sed/manual/html_node/ERE-syntax.html
|
107 | """
|
108 | return pyutil.BackslashEscape(s, ERE_META_CHARS)
|
109 |
|
110 |
|
111 | def GlobUnescape(s):
|
112 | # type: (str) -> str
|
113 | """Remove glob escaping from a string.
|
114 |
|
115 | Used when there is no glob match.
|
116 | TODO: Can probably get rid of this, as long as you save the original word.
|
117 |
|
118 | Complicated example: 'a*b'*.py, which will be escaped to a\*b*.py. So in
|
119 | word_eval _JoinElideEscape and EvalWordToString you have to build two
|
120 | 'parallel' strings -- one escaped and one not.
|
121 | """
|
122 | unescaped = [] # type: List[int]
|
123 | i = 0
|
124 | n = len(s)
|
125 | while i < n:
|
126 | c = mylib.ByteAt(s, i)
|
127 |
|
128 | if mylib.ByteEquals(c, '\\') and i != n - 1:
|
129 | # Suppressed this to fix bug #698, #628 is still there.
|
130 | assert i != n - 1, 'Trailing backslash: %r' % s
|
131 | i += 1
|
132 | c2 = mylib.ByteAt(s, i)
|
133 |
|
134 | if mylib.ByteInSet(c2, GLOB_META_CHARS):
|
135 | unescaped.append(c2)
|
136 | else:
|
137 | raise AssertionError("Unexpected escaped character %r" % c2)
|
138 | else:
|
139 | unescaped.append(c)
|
140 | i += 1
|
141 | return mylib.JoinBytes(unescaped)
|
142 |
|
143 |
|
144 | # For ${x//foo*/y}, we need to glob patterns, but fnmatch doesn't give you the
|
145 | # positions of matches. So we convert globs to regexps.
|
146 |
|
147 | # Problems:
|
148 | # - What about unicode? Do we have to set any global variables? We want it to
|
149 | # always use utf-8?
|
150 |
|
151 |
|
152 | class _GlobParser(object):
|
153 |
|
154 | def __init__(self, lexer):
|
155 | # type: (SimpleLexer) -> None
|
156 | self.lexer = lexer
|
157 | self.token_type = Id.Undefined_Tok
|
158 | self.token_val = ''
|
159 | self.warnings = [] # type: List[str]
|
160 |
|
161 | def _Next(self):
|
162 | # type: () -> None
|
163 | """Move to the next token."""
|
164 | self.token_type, self.token_val = self.lexer.Next()
|
165 |
|
166 | def _ParseCharClass(self):
|
167 | # type: () -> List[glob_part_t]
|
168 | """
|
169 | Returns:
|
170 | a CharClass if the parse succeeds, or a Literal if fails. In the latter
|
171 | case, we also append a warning.
|
172 | """
|
173 | first_token = glob_part.Literal(self.token_type, self.token_val)
|
174 | balance = 1 # We already saw a [
|
175 | tokens = [] # type: List[Tuple[Id_t, str]]
|
176 |
|
177 | # NOTE: There is a special rule where []] and [[] are valid globs. Also
|
178 | # [^[] and sometimes [^]], although that one is ambiguous!
|
179 | # And [[:space:]] and [[.class.]] has to be taken into account too. I'm
|
180 | # punting on this now because the rule isn't clear and consistent between
|
181 | # shells.
|
182 |
|
183 | while True:
|
184 | self._Next()
|
185 |
|
186 | if self.token_type == Id.Eol_Tok:
|
187 | # TODO: location info
|
188 | self.warnings.append(
|
189 | 'Malformed character class; treating as literal')
|
190 | parts = [first_token] # type: List[glob_part_t]
|
191 | for (id_, s) in tokens:
|
192 | parts.append(glob_part.Literal(id_, s))
|
193 | return parts
|
194 |
|
195 | if self.token_type == Id.Glob_LBracket:
|
196 | balance += 1
|
197 | elif self.token_type == Id.Glob_RBracket:
|
198 | balance -= 1
|
199 |
|
200 | if balance == 0:
|
201 | break
|
202 | tokens.append(
|
203 | (self.token_type, self.token_val)) # Don't append the last ]
|
204 |
|
205 | negated = False
|
206 | if len(tokens):
|
207 | id1, _ = tokens[0]
|
208 | # NOTE: Both ! and ^ work for negation in globs
|
209 | # https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html#Pattern-Matching
|
210 | # TODO: Warn about the one that's not recommended?
|
211 | if id1 in (Id.Glob_Bang, Id.Glob_Caret):
|
212 | negated = True
|
213 | tokens = tokens[1:]
|
214 | strs = [s for _, s in tokens]
|
215 | return [glob_part.CharClass(negated, strs)]
|
216 |
|
217 | def Parse(self):
|
218 | # type: () -> Tuple[List[glob_part_t], List[str]]
|
219 | """
|
220 | Returns:
|
221 | regex string (or None if it's not a glob)
|
222 | A list of warnings about the syntax
|
223 | """
|
224 | parts = [] # type: List[glob_part_t]
|
225 |
|
226 | while True:
|
227 | self._Next()
|
228 | id_ = self.token_type
|
229 | s = self.token_val
|
230 |
|
231 | #log('%s %r', self.token_type, self.token_val)
|
232 | if id_ == Id.Eol_Tok:
|
233 | break
|
234 |
|
235 | if id_ in (Id.Glob_Star, Id.Glob_QMark):
|
236 | parts.append(glob_part.Operator(id_))
|
237 |
|
238 | elif id_ == Id.Glob_LBracket:
|
239 | # Could return a Literal or a CharClass
|
240 | parts.extend(self._ParseCharClass())
|
241 |
|
242 | else: # Glob_{Bang,Caret,CleanLiterals,OtherLiteral,RBracket,EscapedChar,
|
243 | # BadBackslash}
|
244 | parts.append(glob_part.Literal(id_, s))
|
245 |
|
246 | # Also check for warnings. TODO: location info.
|
247 | if id_ == Id.Glob_RBracket:
|
248 | self.warnings.append('Got unescaped right bracket')
|
249 | if id_ == Id.Glob_BadBackslash:
|
250 | self.warnings.append('Got unescaped trailing backslash')
|
251 |
|
252 | return parts, self.warnings
|
253 |
|
254 |
|
255 | _REGEX_CHARS_TO_ESCAPE = '.|^$()+*?[]{}\\'
|
256 |
|
257 |
|
258 | def _GenerateERE(parts):
|
259 | # type: (List[glob_part_t]) -> str
|
260 | out = [] # type: List[str]
|
261 |
|
262 | for part in parts:
|
263 | tag = part.tag()
|
264 | UP_part = part
|
265 |
|
266 | if tag == glob_part_e.Literal:
|
267 | part = cast(glob_part.Literal, UP_part)
|
268 | if part.id == Id.Glob_EscapedChar:
|
269 | assert len(part.s) == 2, part.s
|
270 | # The user could have escaped a char that doesn't need regex escaping,
|
271 | # like \b or something.
|
272 | c = part.s[1]
|
273 | if c in _REGEX_CHARS_TO_ESCAPE:
|
274 | out.append('\\')
|
275 | out.append(c)
|
276 |
|
277 | # ! is only for char class
|
278 | elif part.id in (Id.Glob_CleanLiterals, Id.Glob_Bang):
|
279 | out.append(part.s) # e.g. 'py' doesn't need to be escaped
|
280 |
|
281 | # ^ is only for char class
|
282 | elif part.id in (Id.Glob_OtherLiteral, Id.Glob_Caret):
|
283 | assert len(part.s) == 1, part.s
|
284 | c = part.s
|
285 | if c in _REGEX_CHARS_TO_ESCAPE:
|
286 | out.append('\\')
|
287 | out.append(c)
|
288 |
|
289 | # These are UNMATCHED ones not parsed in a glob class
|
290 | elif part.id == Id.Glob_LBracket:
|
291 | out.append('\\[')
|
292 |
|
293 | elif part.id == Id.Glob_RBracket:
|
294 | out.append('\\]')
|
295 |
|
296 | elif part.id == Id.Glob_BadBackslash:
|
297 | out.append('\\\\')
|
298 |
|
299 | elif part.id == Id.Glob_Caret:
|
300 | out.append('^')
|
301 |
|
302 | else:
|
303 | raise AssertionError(part.id)
|
304 |
|
305 | elif tag == glob_part_e.Operator:
|
306 | part = cast(glob_part.Operator, UP_part)
|
307 | if part.op_id == Id.Glob_QMark:
|
308 | out.append('.')
|
309 | elif part.op_id == Id.Glob_Star:
|
310 | out.append('.*')
|
311 | else:
|
312 | raise AssertionError()
|
313 |
|
314 | elif tag == glob_part_e.CharClass:
|
315 | part = cast(glob_part.CharClass, UP_part)
|
316 | out.append('[')
|
317 | if part.negated:
|
318 | out.append('^')
|
319 |
|
320 | # Important: the character class is LITERALLY preserved, because we
|
321 | # assume glob char classes are EXACTLY the same as regex char classes,
|
322 | # including the escaping rules.
|
323 | #
|
324 | # TWO WEIRD EXCEPTIONS:
|
325 | # \- is moved to the end as '-'.
|
326 | # In GNU libc, [0\-9] ODDLY has a range starting with \ ! But we
|
327 | # want a literal, and the POSIX way to do that is to put it at the end.
|
328 | # \] is moved to the FRONT as ]
|
329 |
|
330 | good = [] # type: List[str]
|
331 |
|
332 | literal_hyphen = False
|
333 | literal_rbracket = False
|
334 |
|
335 | for s in part.strs:
|
336 | if s == '\-':
|
337 | literal_hyphen = True
|
338 | continue
|
339 | if s == '\]':
|
340 | literal_rbracket = True
|
341 | continue
|
342 | good.append(s)
|
343 |
|
344 | if literal_rbracket:
|
345 | out.append(']')
|
346 |
|
347 | out.extend(good)
|
348 |
|
349 | if literal_hyphen:
|
350 | out.append('-')
|
351 |
|
352 | out.append(']')
|
353 |
|
354 | return ''.join(out)
|
355 |
|
356 |
|
357 | def GlobToERE(pat):
|
358 | # type: (str) -> Tuple[str, List[str]]
|
359 | lexer = match.GlobLexer(pat)
|
360 | p = _GlobParser(lexer)
|
361 | parts, warnings = p.Parse()
|
362 |
|
363 | # Vestigial: if there is nothing like * ? or [abc], then the whole string is
|
364 | # a literal, and we could use a more efficient mechanism.
|
365 | # But we would have to DEQUOTE before doing that.
|
366 | if 0:
|
367 | is_glob = False
|
368 | for p in parts:
|
369 | if p.tag in (glob_part_e.Operator, glob_part_e.CharClass):
|
370 | is_glob = True
|
371 | if 0:
|
372 | log('GlobToERE()')
|
373 | for p in parts:
|
374 | log(' %s', p)
|
375 |
|
376 | regex = _GenerateERE(parts)
|
377 | #log('pat %s -> regex %s', pat, regex)
|
378 | return regex, warnings
|
379 |
|
380 |
|
381 | # Notes for implementing extglob
|
382 | # - libc glob() doesn't have any extension!
|
383 | # - Nix stdenv uses !(foo) and @(foo|bar)
|
384 | # - can we special case these for now?
|
385 | # - !(foo|bar) -- change it to *, and then just do fnmatch() to filter the
|
386 | # result!
|
387 | # - Actually I guess we can do that for all of them. That seems fine.
|
388 | # - But we have to get the statically parsed arg in here?
|
389 | # - or do dynamic parsing
|
390 | # - LooksLikeGlob() would have to respect extglob! ugh!
|
391 | # - See 2 calls in osh/word_eval.py
|
392 |
|
393 |
|
394 | class Globber(object):
|
395 |
|
396 | def __init__(self, exec_opts):
|
397 | # type: (optview.Exec) -> None
|
398 | self.exec_opts = exec_opts
|
399 |
|
400 | # Other unimplemented bash options:
|
401 | #
|
402 | # dotglob dotfiles are matched
|
403 | # globstar ** for directories
|
404 | # globasciiranges ascii or unicode char classes (unicode by default)
|
405 | # nocaseglob
|
406 | # extglob the @() !() syntax -- libc helps us with fnmatch(), but
|
407 | # not glob().
|
408 | #
|
409 | # NOTE: Bash also respects the GLOBIGNORE variable, but no other shells
|
410 | # do. Could a default GLOBIGNORE to ignore flags on the file system be
|
411 | # part of the security solution? It doesn't seem totally sound.
|
412 |
|
413 | def _Glob(self, arg, out):
|
414 | # type: (str, List[str]) -> int
|
415 | try:
|
416 | results = libc.glob(arg)
|
417 | except RuntimeError as e:
|
418 | # These errors should be rare: I/O error, out of memory, or unknown
|
419 | # There are no syntax errors. (But see comment about globerr() in
|
420 | # native/libc.c.)
|
421 | # note: MyPy doesn't know RuntimeError has e.message (and e.args)
|
422 | msg = e.message # type: str
|
423 | print_stderr("Error expanding glob %r: %s" % (arg, msg))
|
424 | raise
|
425 | #log('glob %r -> %r', arg, g)
|
426 |
|
427 | n = len(results)
|
428 | if n: # Something matched
|
429 | # Omit files starting with -
|
430 | # dashglob turned OFF with shopt -s oil:upgrade.
|
431 | if not self.exec_opts.dashglob():
|
432 | tmp = [s for s in results if not s.startswith('-')]
|
433 | results = tmp # idiom to work around mycpp limitation
|
434 | n = len(results)
|
435 |
|
436 | out.extend(results)
|
437 | return n
|
438 |
|
439 | return 0
|
440 |
|
441 | def Expand(self, arg, out):
|
442 | # type: (str, List[str]) -> int
|
443 | """Given a string that could be a glob, append a list of strings to
|
444 | 'out'.
|
445 |
|
446 | Returns:
|
447 | Number of items appended, or -1 for fatal failglob error.
|
448 | """
|
449 | if self.exec_opts.noglob():
|
450 | # we didn't glob escape it in osh/word_eval.py
|
451 | out.append(arg)
|
452 | return 1
|
453 |
|
454 | n = self._Glob(arg, out)
|
455 | if n:
|
456 | return n
|
457 |
|
458 | # Nothing matched
|
459 | if self.exec_opts.failglob():
|
460 | return -1
|
461 |
|
462 | if self.exec_opts.nullglob():
|
463 | return 0
|
464 | else:
|
465 | # Return the original string
|
466 | out.append(GlobUnescape(arg))
|
467 | return 1
|
468 |
|
469 | def ExpandExtended(self, glob_pat, fnmatch_pat, out):
|
470 | # type: (str, str, List[str]) -> int
|
471 | if self.exec_opts.noglob():
|
472 | # Return the fnmatch_pat. Note: this means we turn ,() into @(), and
|
473 | # there is extra \ escaping compared with bash and mksh. OK for now
|
474 | out.append(fnmatch_pat)
|
475 | return 1
|
476 |
|
477 | tmp = [] # type: List[str]
|
478 | self._Glob(glob_pat, tmp)
|
479 | filtered = [s for s in tmp if libc.fnmatch(fnmatch_pat, s)]
|
480 | n = len(filtered)
|
481 |
|
482 | if n:
|
483 | out.extend(filtered)
|
484 | return n
|
485 |
|
486 | if self.exec_opts.failglob():
|
487 | return -1 # nothing matched
|
488 |
|
489 | if self.exec_opts.nullglob():
|
490 | return 0
|
491 | else:
|
492 | # See comment above
|
493 | out.append(GlobUnescape(fnmatch_pat))
|
494 | return 1
|