OILS / builtin / func_misc.py View on Github | oilshell.org

743 lines, 424 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str)
9
10from core import error
11from core import num
12from core import state
13from core import ui
14from core import vm
15from data_lang import j8
16from frontend import match
17from frontend import typed_args
18from mycpp import mops
19from mycpp import mylib
20from mycpp.mylib import NewDict, iteritems, log, tagswitch
21from ysh import expr_eval
22from ysh import val_ops
23
24from typing import TYPE_CHECKING, Dict, List, cast
25if TYPE_CHECKING:
26 from osh import glob_
27 from osh import split
28
29_ = log
30
31
32class Len(vm._Callable):
33
34 def __init__(self):
35 # type: () -> None
36 pass
37
38 def Call(self, rd):
39 # type: (typed_args.Reader) -> value_t
40
41 x = rd.PosValue()
42 rd.Done()
43
44 UP_x = x
45 with tagswitch(x) as case:
46 if case(value_e.List):
47 x = cast(value.List, UP_x)
48 return num.ToBig(len(x.items))
49
50 elif case(value_e.Dict):
51 x = cast(value.Dict, UP_x)
52 return num.ToBig(len(x.d))
53
54 elif case(value_e.Str):
55 x = cast(value.Str, UP_x)
56 return num.ToBig(len(x.s))
57
58 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
59 rd.BlamePos())
60
61
62class Type(vm._Callable):
63
64 def __init__(self):
65 # type: () -> None
66 pass
67
68 def Call(self, rd):
69 # type: (typed_args.Reader) -> value_t
70
71 val = rd.PosValue()
72 rd.Done()
73
74 return value.Str(ui.ValType(val))
75
76
77class Join(vm._Callable):
78 """Both free function join() and List->join() method."""
79
80 def __init__(self):
81 # type: () -> None
82 pass
83
84 def Call(self, rd):
85 # type: (typed_args.Reader) -> value_t
86
87 li = rd.PosList()
88 delim = rd.OptionalStr(default_='')
89 rd.Done()
90
91 strs = [] # type: List[str]
92 for i, el in enumerate(li):
93 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
94
95 return value.Str(delim.join(strs))
96
97
98class Maybe(vm._Callable):
99
100 def __init__(self):
101 # type: () -> None
102 pass
103
104 def Call(self, rd):
105 # type: (typed_args.Reader) -> value_t
106
107 val = rd.PosValue()
108 rd.Done()
109
110 if val == value.Null:
111 return value.List([])
112
113 s = val_ops.ToStr(
114 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
115 rd.LeftParenToken())
116 if len(s):
117 return value.List([val]) # use val to avoid needlessly copy
118
119 return value.List([])
120
121
122class Bool(vm._Callable):
123
124 def __init__(self):
125 # type: () -> None
126 pass
127
128 def Call(self, rd):
129 # type: (typed_args.Reader) -> value_t
130
131 val = rd.PosValue()
132 rd.Done()
133
134 return value.Bool(val_ops.ToBool(val))
135
136
137class Int(vm._Callable):
138
139 def __init__(self):
140 # type: () -> None
141 pass
142
143 def Call(self, rd):
144 # type: (typed_args.Reader) -> value_t
145
146 val = rd.PosValue()
147 rd.Done()
148
149 UP_val = val
150 with tagswitch(val) as case:
151 if case(value_e.Int):
152 return val
153
154 elif case(value_e.Bool):
155 val = cast(value.Bool, UP_val)
156 return value.Int(mops.FromBool(val.b))
157
158 elif case(value_e.Float):
159 val = cast(value.Float, UP_val)
160 return value.Int(mops.FromFloat(val.f))
161
162 elif case(value_e.Str):
163 val = cast(value.Str, UP_val)
164 if not match.LooksLikeInteger(val.s):
165 raise error.Expr('Cannot convert %s to Int' % val.s,
166 rd.BlamePos())
167
168 return value.Int(mops.FromStr(val.s))
169
170 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
171 rd.BlamePos())
172
173
174class Float(vm._Callable):
175
176 def __init__(self):
177 # type: () -> None
178 pass
179
180 def Call(self, rd):
181 # type: (typed_args.Reader) -> value_t
182
183 val = rd.PosValue()
184 rd.Done()
185
186 UP_val = val
187 with tagswitch(val) as case:
188 if case(value_e.Int):
189 val = cast(value.Int, UP_val)
190 return value.Float(mops.ToFloat(val.i))
191
192 elif case(value_e.Float):
193 return val
194
195 elif case(value_e.Str):
196 val = cast(value.Str, UP_val)
197 if not match.LooksLikeFloat(val.s):
198 raise error.Expr('Cannot convert %s to Float' % val.s,
199 rd.BlamePos())
200
201 return value.Float(float(val.s))
202
203 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
204 rd.BlamePos())
205
206
207class Str_(vm._Callable):
208
209 def __init__(self):
210 # type: () -> None
211 pass
212
213 def Call(self, rd):
214 # type: (typed_args.Reader) -> value_t
215
216 val = rd.PosValue()
217 rd.Done()
218
219 # TODO: Should we call Stringify here? That would handle Eggex.
220
221 UP_val = val
222 with tagswitch(val) as case:
223 if case(value_e.Int):
224 val = cast(value.Int, UP_val)
225 return value.Str(mops.ToStr(val.i))
226
227 elif case(value_e.Float):
228 val = cast(value.Float, UP_val)
229 return value.Str(str(val.f))
230
231 elif case(value_e.Str):
232 return val
233
234 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
235 rd.BlamePos())
236
237
238class List_(vm._Callable):
239
240 def __init__(self):
241 # type: () -> None
242 pass
243
244 def Call(self, rd):
245 # type: (typed_args.Reader) -> value_t
246
247 val = rd.PosValue()
248 rd.Done()
249
250 l = [] # type: List[value_t]
251 it = None # type: val_ops.Iterator
252 UP_val = val
253 with tagswitch(val) as case:
254 if case(value_e.List):
255 val = cast(value.List, UP_val)
256 it = val_ops.ListIterator(val)
257
258 elif case(value_e.Dict):
259 val = cast(value.Dict, UP_val)
260 it = val_ops.DictIterator(val)
261
262 elif case(value_e.Range):
263 val = cast(value.Range, UP_val)
264 it = val_ops.RangeIterator(val)
265
266 else:
267 raise error.TypeErr(val,
268 'list() expected Dict, List, or Range',
269 rd.BlamePos())
270
271 assert it is not None
272 while True:
273 first = it.FirstValue()
274 if first is None:
275 break
276 l.append(first)
277 it.Next()
278
279 return value.List(l)
280
281
282class Dict_(vm._Callable):
283
284 def __init__(self):
285 # type: () -> None
286 pass
287
288 def Call(self, rd):
289 # type: (typed_args.Reader) -> value_t
290
291 val = rd.PosValue()
292 rd.Done()
293
294 UP_val = val
295 with tagswitch(val) as case:
296 if case(value_e.Dict):
297 d = NewDict() # type: Dict[str, value_t]
298 val = cast(value.Dict, UP_val)
299 for k, v in iteritems(val.d):
300 d[k] = v
301
302 return value.Dict(d)
303
304 elif case(value_e.BashAssoc):
305 d = NewDict()
306 val = cast(value.BashAssoc, UP_val)
307 for k, s in iteritems(val.d):
308 d[k] = value.Str(s)
309
310 return value.Dict(d)
311
312 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
313 rd.BlamePos())
314
315
316class Runes(vm._Callable):
317
318 def __init__(self):
319 # type: () -> None
320 pass
321
322 def Call(self, rd):
323 # type: (typed_args.Reader) -> value_t
324 return value.Null
325
326
327class EncodeRunes(vm._Callable):
328
329 def __init__(self):
330 # type: () -> None
331 pass
332
333 def Call(self, rd):
334 # type: (typed_args.Reader) -> value_t
335 return value.Null
336
337
338class Bytes(vm._Callable):
339
340 def __init__(self):
341 # type: () -> None
342 pass
343
344 def Call(self, rd):
345 # type: (typed_args.Reader) -> value_t
346 return value.Null
347
348
349class EncodeBytes(vm._Callable):
350
351 def __init__(self):
352 # type: () -> None
353 pass
354
355 def Call(self, rd):
356 # type: (typed_args.Reader) -> value_t
357 return value.Null
358
359
360class Split(vm._Callable):
361
362 def __init__(self, splitter):
363 # type: (split.SplitContext) -> None
364 vm._Callable.__init__(self)
365 self.splitter = splitter
366
367 def Call(self, rd):
368 # type: (typed_args.Reader) -> value_t
369 s = rd.PosStr()
370
371 ifs = rd.OptionalStr()
372
373 rd.Done()
374
375 l = [
376 value.Str(elem)
377 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
378 ] # type: List[value_t]
379 return value.List(l)
380
381
382class Glob(vm._Callable):
383
384 def __init__(self, globber):
385 # type: (glob_.Globber) -> None
386 vm._Callable.__init__(self)
387 self.globber = globber
388
389 def Call(self, rd):
390 # type: (typed_args.Reader) -> value_t
391 s = rd.PosStr()
392 rd.Done()
393
394 out = [] # type: List[str]
395 self.globber._Glob(s, out)
396
397 l = [value.Str(elem) for elem in out] # type: List[value_t]
398 return value.List(l)
399
400
401class Shvar_get(vm._Callable):
402 """Look up with dynamic scope."""
403
404 def __init__(self, mem):
405 # type: (state.Mem) -> None
406 vm._Callable.__init__(self)
407 self.mem = mem
408
409 def Call(self, rd):
410 # type: (typed_args.Reader) -> value_t
411 name = rd.PosStr()
412 rd.Done()
413 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
414
415
416class GetVar(vm._Callable):
417 """Look up normal scoping rules."""
418
419 def __init__(self, mem):
420 # type: (state.Mem) -> None
421 vm._Callable.__init__(self)
422 self.mem = mem
423
424 def Call(self, rd):
425 # type: (typed_args.Reader) -> value_t
426 name = rd.PosStr()
427 rd.Done()
428 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
429
430
431class Assert(vm._Callable):
432
433 def __init__(self):
434 # type: () -> None
435 pass
436
437 def Call(self, rd):
438 # type: (typed_args.Reader) -> value_t
439
440 val = rd.PosValue()
441
442 msg = rd.OptionalStr(default_='')
443
444 rd.Done()
445
446 if not val_ops.ToBool(val):
447 raise error.AssertionErr(msg, rd.LeftParenToken())
448
449 return value.Null
450
451
452class EvalExpr(vm._Callable):
453
454 def __init__(self, expr_ev):
455 # type: (expr_eval.ExprEvaluator) -> None
456 self.expr_ev = expr_ev
457
458 def Call(self, rd):
459 # type: (typed_args.Reader) -> value_t
460 lazy = rd.PosExpr()
461 rd.Done()
462
463 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
464
465 return result
466
467
468class ToJson8(vm._Callable):
469
470 def __init__(self, is_j8):
471 # type: (bool) -> None
472 self.is_j8 = is_j8
473
474 def Call(self, rd):
475 # type: (typed_args.Reader) -> value_t
476
477 val = rd.PosValue()
478 space = mops.BigTruncate(rd.NamedInt('space', 0))
479 rd.Done()
480
481 # Convert from external JS-like API to internal API.
482 if space <= 0:
483 indent = -1
484 else:
485 indent = space
486
487 buf = mylib.BufWriter()
488 try:
489 if self.is_j8:
490 j8.PrintMessage(val, buf, indent)
491 else:
492 j8.PrintJsonMessage(val, buf, indent)
493 except error.Encode as e:
494 # status code 4 is special, for encode/decode errors.
495 raise error.Structured(4, e.Message(), rd.LeftParenToken())
496
497 return value.Str(buf.getvalue())
498
499
500class FromJson8(vm._Callable):
501
502 def __init__(self, is_j8):
503 # type: (bool) -> None
504 self.is_j8 = is_j8
505
506 def Call(self, rd):
507 # type: (typed_args.Reader) -> value_t
508
509 s = rd.PosStr()
510 rd.Done()
511
512 p = j8.Parser(s, self.is_j8)
513 try:
514 val = p.ParseValue()
515 except error.Decode as e:
516 # Right now I'm not exposing the original string, because that
517 # could lead to a memory leak in the _error Dict.
518 # The message quotes part of the string, and we could improve
519 # that. We could have a substring with context.
520 props = {
521 'start_pos': num.ToBig(e.start_pos),
522 'end_pos': num.ToBig(e.end_pos),
523 } # type: Dict[str, value_t]
524 # status code 4 is special, for encode/decode errors.
525 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
526
527 return val
528
529
530class BashArrayToSparse(vm._Callable):
531 """
532 value.BashArray -> value.SparseArray, for testing
533 """
534
535 def __init__(self):
536 # type: () -> None
537 pass
538
539 def Call(self, rd):
540 # type: (typed_args.Reader) -> value_t
541
542 strs = rd.PosBashArray()
543 rd.Done()
544
545 d = {} # type: Dict[mops.BigInt, str]
546 max_index = mops.MINUS_ONE # max index for empty array
547 for i, s in enumerate(strs):
548 if s is not None:
549 big_i = mops.IntWiden(i)
550 d[big_i] = s
551 if mops.Greater(big_i, max_index):
552 max_index = big_i
553
554 return value.SparseArray(d, max_index)
555
556
557class DictToSparse(vm._Callable):
558 """
559 value.Dict -> value.SparseArray, for testing
560 """
561
562 def __init__(self):
563 # type: () -> None
564 pass
565
566 def Call(self, rd):
567 # type: (typed_args.Reader) -> value_t
568
569 d = rd.PosDict()
570 rd.Done()
571
572 blame_tok = rd.LeftParenToken()
573
574 mydict = {} # type: Dict[mops.BigInt, str]
575 for k, v in iteritems(d):
576 i = mops.FromStr(k)
577 s = val_ops.ToStr(v, 'expected str', blame_tok)
578
579 mydict[i] = s
580
581 max_index = mops.MINUS_ONE # TODO:
582 return value.SparseArray(mydict, max_index)
583
584
585class SparseOp(vm._Callable):
586 """
587 All ops on value.SparseArray, for testing performance
588 """
589
590 def __init__(self):
591 # type: () -> None
592 pass
593
594 def Call(self, rd):
595 # type: (typed_args.Reader) -> value_t
596
597 sp = rd.PosSparseArray()
598 d = sp.d
599 #i = mops.BigTruncate(rd.PosInt())
600 op_name = rd.PosStr()
601
602 no_str = None # type: str
603
604 if op_name == 'len': # ${#a[@]}
605 rd.Done()
606 return num.ToBig(len(d))
607
608 elif op_name == 'get': # ${a[42]}
609 index = rd.PosInt()
610 rd.Done()
611
612 s = d.get(index)
613 if s is None:
614 return value.Null
615 else:
616 return value.Str(s)
617
618 elif op_name == 'set': # a[42]=foo
619 index = rd.PosInt()
620 s = rd.PosStr()
621 rd.Done()
622
623 d[index] = s
624
625 if mops.Greater(index, sp.max_index):
626 sp.max_index = index
627
628 return value.Int(mops.ZERO)
629
630 elif op_name == 'unset': # unset 'a[1]'
631 index = rd.PosInt()
632 rd.Done()
633
634 mylib.dict_erase(d, index)
635
636 max_index = mops.MINUS_ONE # Note: this works if d is not empty
637 for i1 in d:
638 if mops.Greater(i1, max_index): # i1 > max_index
639 max_index = i1
640 sp.max_index = max_index
641
642 return value.Int(mops.ZERO)
643
644 elif op_name == 'subst': # "${a[@]}"
645 # Algorithm to expand a Dict[BigInt, Str]
646 #
647 # 1. Copy the integer keys into a new List
648 # 2. Sort them in numeric order
649 # 3. Create a List[str] that's the same size as the keys
650 # 4. Loop through sorted keys, look up value, and populate list
651 #
652 # There is another possible algorithm:
653 #
654 # 1. Copy the VALUES into a new list
655 # 2. Somehow sort them by the CORRESPONDING key, which depends on
656 # Slab<> POSITION. I think this does not fit within the
657 # std::sort() model. I think we would have to write a little custom
658 # sort algorithm.
659
660 keys = d.keys()
661 mylib.BigIntSort(keys)
662 # Pre-allocate
663 items = [no_str] * len(d) # type: List[str]
664 j = 0
665 for i in keys:
666 s = d.get(i)
667 assert s is not None
668 items[j] = s
669 j += 1
670 return value.BashArray(items)
671
672 elif op_name == 'keys': # "${!a[@]}"
673 keys = d.keys()
674 mylib.BigIntSort(keys)
675 items = [mops.ToStr(k) for k in keys]
676
677 # TODO: return SparseArray
678 return value.BashArray(items)
679
680 elif op_name == 'slice': # "${a[@]:0:5}"
681 start = rd.PosInt()
682 end = rd.PosInt()
683 rd.Done()
684
685 n = mops.BigTruncate(mops.Sub(end, start))
686 #log('start %d - end %d', start.i, end.i)
687
688 # Pre-allocate
689 items2 = [no_str] * n # type: List[str]
690
691 # Iterate from start to end. Note that this algorithm is
692 # theoretically slower than bash in the case where the array is
693 # sparse (in the part selected by the slice)
694 #
695 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
696 # elements, OSH will iterate through 999 integers and do 999 dict
697 # lookups, while bash will follow 3 pointers.
698 #
699 # However, in practice, I think iterating through integers is
700 # cheap.
701
702 j = 0
703 i = start
704 while mops.Greater(end, i): # i < end
705 s = d.get(i)
706 #log('s %s', s)
707 if s is not None:
708 items2[j] = s
709 j += 1
710
711 i = mops.Add(i, mops.ONE) # i += 1
712
713 # TODO: return SparseArray
714 return value.BashArray(items2)
715
716 elif op_name == 'append': # a+=(x y)
717 strs = rd.PosBashArray()
718
719 # TODO: We can maintain the max index in the value.SparseArray(),
720 # so that it's O(1) to append rather than O(n)
721 # - Update on 'set' is O(1)
722 # - Update on 'unset' is potentially O(n)
723
724 if 0:
725 max_index = mops.MINUS_ONE # Note: this works for empty arrays
726 for i1 in d:
727 if mops.Greater(i1, max_index): # i1 > max_index
728 max_index = i1
729 else:
730 max_index = sp.max_index
731
732 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
733 for s in strs:
734 d[i2] = s
735 i2 = mops.Add(i2, mops.ONE) # i2 += 1
736
737 # sp.max_index += len(strs)
738 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
739 return value.Int(mops.ZERO)
740
741 else:
742 print('Invalid SparseArray operation %r' % op_name)
743 return value.Int(mops.ZERO)