OILS / builtin / func_misc.py View on Github | oilshell.org

751 lines, 426 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str)
9
10from core import error
11from core import num
12from core import state
13from core import ui
14from core import vm
15from data_lang import j8
16from frontend import match
17from frontend import typed_args
18from mycpp import mops
19from mycpp import mylib
20from mycpp.mylib import NewDict, iteritems, log, tagswitch
21from ysh import expr_eval
22from ysh import val_ops
23
24from typing import TYPE_CHECKING, Dict, List, cast
25if TYPE_CHECKING:
26 from osh import glob_
27 from osh import split
28
29_ = log
30
31
32class Len(vm._Callable):
33
34 def __init__(self):
35 # type: () -> None
36 pass
37
38 def Call(self, rd):
39 # type: (typed_args.Reader) -> value_t
40
41 x = rd.PosValue()
42 rd.Done()
43
44 UP_x = x
45 with tagswitch(x) as case:
46 if case(value_e.List):
47 x = cast(value.List, UP_x)
48 return num.ToBig(len(x.items))
49
50 elif case(value_e.Dict):
51 x = cast(value.Dict, UP_x)
52 return num.ToBig(len(x.d))
53
54 elif case(value_e.Str):
55 x = cast(value.Str, UP_x)
56 return num.ToBig(len(x.s))
57
58 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
59 rd.BlamePos())
60
61
62class Type(vm._Callable):
63
64 def __init__(self):
65 # type: () -> None
66 pass
67
68 def Call(self, rd):
69 # type: (typed_args.Reader) -> value_t
70
71 val = rd.PosValue()
72 rd.Done()
73
74 return value.Str(ui.ValType(val))
75
76
77class Repeat(vm._Callable):
78
79 def __init__(self):
80 # type: () -> None
81 pass
82
83 def Call(self, rd):
84 # type: (typed_args.Reader) -> value_t
85 return value.Null
86
87
88class Join(vm._Callable):
89 """Both free function join() and List->join() method."""
90
91 def __init__(self):
92 # type: () -> None
93 pass
94
95 def Call(self, rd):
96 # type: (typed_args.Reader) -> value_t
97
98 li = rd.PosList()
99 delim = rd.OptionalStr(default_='')
100 rd.Done()
101
102 strs = [] # type: List[str]
103 for i, el in enumerate(li):
104 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
105
106 return value.Str(delim.join(strs))
107
108
109class Maybe(vm._Callable):
110
111 def __init__(self):
112 # type: () -> None
113 pass
114
115 def Call(self, rd):
116 # type: (typed_args.Reader) -> value_t
117
118 val = rd.PosValue()
119 rd.Done()
120
121 if val == value.Null:
122 return value.List([])
123
124 s = val_ops.ToStr(
125 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
126 rd.LeftParenToken())
127 if len(s):
128 return value.List([val]) # use val to avoid needlessly copy
129
130 return value.List([])
131
132
133class Bool(vm._Callable):
134
135 def __init__(self):
136 # type: () -> None
137 pass
138
139 def Call(self, rd):
140 # type: (typed_args.Reader) -> value_t
141
142 val = rd.PosValue()
143 rd.Done()
144
145 return value.Bool(val_ops.ToBool(val))
146
147
148class Int(vm._Callable):
149
150 def __init__(self):
151 # type: () -> None
152 pass
153
154 def Call(self, rd):
155 # type: (typed_args.Reader) -> value_t
156
157 val = rd.PosValue()
158 rd.Done()
159
160 UP_val = val
161 with tagswitch(val) as case:
162 if case(value_e.Int):
163 return val
164
165 elif case(value_e.Bool):
166 val = cast(value.Bool, UP_val)
167 return value.Int(mops.FromBool(val.b))
168
169 elif case(value_e.Float):
170 val = cast(value.Float, UP_val)
171 return value.Int(mops.FromFloat(val.f))
172
173 elif case(value_e.Str):
174 val = cast(value.Str, UP_val)
175 if not match.LooksLikeInteger(val.s):
176 raise error.Expr('Cannot convert %s to Int' % val.s,
177 rd.BlamePos())
178
179 return value.Int(mops.FromStr(val.s))
180
181 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
182 rd.BlamePos())
183
184
185class Float(vm._Callable):
186
187 def __init__(self):
188 # type: () -> None
189 pass
190
191 def Call(self, rd):
192 # type: (typed_args.Reader) -> value_t
193
194 val = rd.PosValue()
195 rd.Done()
196
197 UP_val = val
198 with tagswitch(val) as case:
199 if case(value_e.Int):
200 val = cast(value.Int, UP_val)
201 return value.Float(mops.ToFloat(val.i))
202
203 elif case(value_e.Float):
204 return val
205
206 elif case(value_e.Str):
207 val = cast(value.Str, UP_val)
208 if not match.LooksLikeFloat(val.s):
209 raise error.Expr('Cannot convert %s to Float' % val.s,
210 rd.BlamePos())
211
212 return value.Float(float(val.s))
213
214 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
215 rd.BlamePos())
216
217
218class Str_(vm._Callable):
219
220 def __init__(self):
221 # type: () -> None
222 pass
223
224 def Call(self, rd):
225 # type: (typed_args.Reader) -> value_t
226
227 val = rd.PosValue()
228 rd.Done()
229
230 # TODO: Should we call Stringify here? That would handle Eggex.
231
232 UP_val = val
233 with tagswitch(val) as case:
234 if case(value_e.Int):
235 val = cast(value.Int, UP_val)
236 return value.Str(mops.ToStr(val.i))
237
238 elif case(value_e.Float):
239 val = cast(value.Float, UP_val)
240 return value.Str(str(val.f))
241
242 elif case(value_e.Str):
243 return val
244
245 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
246 rd.BlamePos())
247
248
249class List_(vm._Callable):
250
251 def __init__(self):
252 # type: () -> None
253 pass
254
255 def Call(self, rd):
256 # type: (typed_args.Reader) -> value_t
257
258 val = rd.PosValue()
259 rd.Done()
260
261 l = [] # type: List[value_t]
262 it = None # type: val_ops._ContainerIter
263 UP_val = val
264 with tagswitch(val) as case:
265 if case(value_e.List):
266 val = cast(value.List, UP_val)
267 it = val_ops.ListIterator(val)
268
269 elif case(value_e.Dict):
270 val = cast(value.Dict, UP_val)
271 it = val_ops.DictIterator(val)
272
273 elif case(value_e.Range):
274 val = cast(value.Range, UP_val)
275 it = val_ops.RangeIterator(val)
276
277 else:
278 raise error.TypeErr(val,
279 'list() expected Dict, List, or Range',
280 rd.BlamePos())
281
282 assert it is not None
283 while not it.Done():
284 l.append(it.FirstValue())
285 it.Next()
286
287 return value.List(l)
288
289
290class Dict_(vm._Callable):
291
292 def __init__(self):
293 # type: () -> None
294 pass
295
296 def Call(self, rd):
297 # type: (typed_args.Reader) -> value_t
298
299 val = rd.PosValue()
300 rd.Done()
301
302 UP_val = val
303 with tagswitch(val) as case:
304 if case(value_e.Dict):
305 d = NewDict() # type: Dict[str, value_t]
306 val = cast(value.Dict, UP_val)
307 for k, v in iteritems(val.d):
308 d[k] = v
309
310 return value.Dict(d)
311
312 elif case(value_e.BashAssoc):
313 d = NewDict()
314 val = cast(value.BashAssoc, UP_val)
315 for k, s in iteritems(val.d):
316 d[k] = value.Str(s)
317
318 return value.Dict(d)
319
320 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
321 rd.BlamePos())
322
323
324class Runes(vm._Callable):
325
326 def __init__(self):
327 # type: () -> None
328 pass
329
330 def Call(self, rd):
331 # type: (typed_args.Reader) -> value_t
332 return value.Null
333
334
335class EncodeRunes(vm._Callable):
336
337 def __init__(self):
338 # type: () -> None
339 pass
340
341 def Call(self, rd):
342 # type: (typed_args.Reader) -> value_t
343 return value.Null
344
345
346class Bytes(vm._Callable):
347
348 def __init__(self):
349 # type: () -> None
350 pass
351
352 def Call(self, rd):
353 # type: (typed_args.Reader) -> value_t
354 return value.Null
355
356
357class EncodeBytes(vm._Callable):
358
359 def __init__(self):
360 # type: () -> None
361 pass
362
363 def Call(self, rd):
364 # type: (typed_args.Reader) -> value_t
365 return value.Null
366
367
368class Split(vm._Callable):
369
370 def __init__(self, splitter):
371 # type: (split.SplitContext) -> None
372 vm._Callable.__init__(self)
373 self.splitter = splitter
374
375 def Call(self, rd):
376 # type: (typed_args.Reader) -> value_t
377 s = rd.PosStr()
378
379 ifs = rd.OptionalStr()
380
381 rd.Done()
382
383 l = [
384 value.Str(elem)
385 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
386 ] # type: List[value_t]
387 return value.List(l)
388
389
390class Glob(vm._Callable):
391
392 def __init__(self, globber):
393 # type: (glob_.Globber) -> None
394 vm._Callable.__init__(self)
395 self.globber = globber
396
397 def Call(self, rd):
398 # type: (typed_args.Reader) -> value_t
399 s = rd.PosStr()
400 rd.Done()
401
402 out = [] # type: List[str]
403 self.globber._Glob(s, out)
404
405 l = [value.Str(elem) for elem in out] # type: List[value_t]
406 return value.List(l)
407
408
409class Shvar_get(vm._Callable):
410 """Look up with dynamic scope."""
411
412 def __init__(self, mem):
413 # type: (state.Mem) -> None
414 vm._Callable.__init__(self)
415 self.mem = mem
416
417 def Call(self, rd):
418 # type: (typed_args.Reader) -> value_t
419 name = rd.PosStr()
420 rd.Done()
421 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
422
423
424class GetVar(vm._Callable):
425 """Look up normal scoping rules."""
426
427 def __init__(self, mem):
428 # type: (state.Mem) -> None
429 vm._Callable.__init__(self)
430 self.mem = mem
431
432 def Call(self, rd):
433 # type: (typed_args.Reader) -> value_t
434 name = rd.PosStr()
435 rd.Done()
436 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
437
438
439class Assert(vm._Callable):
440
441 def __init__(self):
442 # type: () -> None
443 pass
444
445 def Call(self, rd):
446 # type: (typed_args.Reader) -> value_t
447
448 val = rd.PosValue()
449
450 msg = rd.OptionalStr(default_='')
451
452 rd.Done()
453
454 if not val_ops.ToBool(val):
455 raise error.AssertionErr(msg, rd.LeftParenToken())
456
457 return value.Null
458
459
460class EvalExpr(vm._Callable):
461
462 def __init__(self, expr_ev):
463 # type: (expr_eval.ExprEvaluator) -> None
464 self.expr_ev = expr_ev
465
466 def Call(self, rd):
467 # type: (typed_args.Reader) -> value_t
468 lazy = rd.PosExpr()
469 rd.Done()
470
471 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
472
473 return result
474
475
476class ToJson8(vm._Callable):
477
478 def __init__(self, is_j8):
479 # type: (bool) -> None
480 self.is_j8 = is_j8
481
482 def Call(self, rd):
483 # type: (typed_args.Reader) -> value_t
484
485 val = rd.PosValue()
486 space = mops.BigTruncate(rd.NamedInt('space', 0))
487 rd.Done()
488
489 # Convert from external JS-like API to internal API.
490 if space <= 0:
491 indent = -1
492 else:
493 indent = space
494
495 buf = mylib.BufWriter()
496 try:
497 if self.is_j8:
498 j8.PrintMessage(val, buf, indent)
499 else:
500 j8.PrintJsonMessage(val, buf, indent)
501 except error.Encode as e:
502 # status code 4 is special, for encode/decode errors.
503 raise error.Structured(4, e.Message(), rd.LeftParenToken())
504
505 return value.Str(buf.getvalue())
506
507
508class FromJson8(vm._Callable):
509
510 def __init__(self, is_j8):
511 # type: (bool) -> None
512 self.is_j8 = is_j8
513
514 def Call(self, rd):
515 # type: (typed_args.Reader) -> value_t
516
517 s = rd.PosStr()
518 rd.Done()
519
520 p = j8.Parser(s, self.is_j8)
521 try:
522 val = p.ParseValue()
523 except error.Decode as e:
524 # Right now I'm not exposing the original string, because that
525 # could lead to a memory leak in the _error Dict.
526 # The message quotes part of the string, and we could improve
527 # that. We could have a substring with context.
528 props = {
529 'start_pos': num.ToBig(e.start_pos),
530 'end_pos': num.ToBig(e.end_pos),
531 } # type: Dict[str, value_t]
532 # status code 4 is special, for encode/decode errors.
533 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
534
535 return val
536
537
538class BashArrayToSparse(vm._Callable):
539 """
540 value.BashArray -> value.SparseArray, for testing
541 """
542
543 def __init__(self):
544 # type: () -> None
545 pass
546
547 def Call(self, rd):
548 # type: (typed_args.Reader) -> value_t
549
550 strs = rd.PosBashArray()
551 rd.Done()
552
553 d = {} # type: Dict[mops.BigInt, str]
554 max_index = mops.MINUS_ONE # max index for empty array
555 for i, s in enumerate(strs):
556 if s is not None:
557 big_i = mops.IntWiden(i)
558 d[big_i] = s
559 if mops.Greater(big_i, max_index):
560 max_index = big_i
561
562 return value.SparseArray(d, max_index)
563
564
565class DictToSparse(vm._Callable):
566 """
567 value.Dict -> value.SparseArray, for testing
568 """
569
570 def __init__(self):
571 # type: () -> None
572 pass
573
574 def Call(self, rd):
575 # type: (typed_args.Reader) -> value_t
576
577 d = rd.PosDict()
578 rd.Done()
579
580 blame_tok = rd.LeftParenToken()
581
582 mydict = {} # type: Dict[mops.BigInt, str]
583 for k, v in iteritems(d):
584 i = mops.FromStr(k)
585 s = val_ops.ToStr(v, 'expected str', blame_tok)
586
587 mydict[i] = s
588
589 max_index = mops.MINUS_ONE # TODO:
590 return value.SparseArray(mydict, max_index)
591
592
593class SparseOp(vm._Callable):
594 """
595 All ops on value.SparseArray, for testing performance
596 """
597
598 def __init__(self):
599 # type: () -> None
600 pass
601
602 def Call(self, rd):
603 # type: (typed_args.Reader) -> value_t
604
605 sp = rd.PosSparseArray()
606 d = sp.d
607 #i = mops.BigTruncate(rd.PosInt())
608 op_name = rd.PosStr()
609
610 no_str = None # type: str
611
612 if op_name == 'len': # ${#a[@]}
613 rd.Done()
614 return num.ToBig(len(d))
615
616 elif op_name == 'get': # ${a[42]}
617 index = rd.PosInt()
618 rd.Done()
619
620 s = d.get(index)
621 if s is None:
622 return value.Null
623 else:
624 return value.Str(s)
625
626 elif op_name == 'set': # a[42]=foo
627 index = rd.PosInt()
628 s = rd.PosStr()
629 rd.Done()
630
631 d[index] = s
632
633 if mops.Greater(index, sp.max_index):
634 sp.max_index = index
635
636 return value.Int(mops.ZERO)
637
638 elif op_name == 'unset': # unset 'a[1]'
639 index = rd.PosInt()
640 rd.Done()
641
642 mylib.dict_erase(d, index)
643
644 max_index = mops.MINUS_ONE # Note: this works if d is not empty
645 for i1 in d:
646 if mops.Greater(i1, max_index): # i1 > max_index
647 max_index = i1
648 sp.max_index = max_index
649
650 return value.Int(mops.ZERO)
651
652 elif op_name == 'subst': # "${a[@]}"
653 # Algorithm to expand a Dict[BigInt, Str]
654 #
655 # 1. Copy the integer keys into a new List
656 # 2. Sort them in numeric order
657 # 3. Create a List[str] that's the same size as the keys
658 # 4. Loop through sorted keys, look up value, and populate list
659 #
660 # There is another possible algorithm:
661 #
662 # 1. Copy the VALUES into a new list
663 # 2. Somehow sort them by the CORRESPONDING key, which depends on
664 # Slab<> POSITION. I think this does not fit within the
665 # std::sort() model. I think we would have to write a little custom
666 # sort algorithm.
667
668 keys = d.keys()
669 mylib.BigIntSort(keys)
670 # Pre-allocate
671 items = [no_str] * len(d) # type: List[str]
672 j = 0
673 for i in keys:
674 s = d.get(i)
675 assert s is not None
676 items[j] = s
677 j += 1
678 return value.BashArray(items)
679
680 elif op_name == 'keys': # "${!a[@]}"
681 keys = d.keys()
682 mylib.BigIntSort(keys)
683 items = [mops.ToStr(k) for k in keys]
684
685 # TODO: return SparseArray
686 return value.BashArray(items)
687
688 elif op_name == 'slice': # "${a[@]:0:5}"
689 start = rd.PosInt()
690 end = rd.PosInt()
691 rd.Done()
692
693 n = mops.BigTruncate(mops.Sub(end, start))
694 #log('start %d - end %d', start.i, end.i)
695
696 # Pre-allocate
697 items2 = [no_str] * n # type: List[str]
698
699 # Iterate from start to end. Note that this algorithm is
700 # theoretically slower than bash in the case where the array is
701 # sparse (in the part selected by the slice)
702 #
703 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
704 # elements, OSH will iterate through 999 integers and do 999 dict
705 # lookups, while bash will follow 3 pointers.
706 #
707 # However, in practice, I think iterating through integers is
708 # cheap.
709
710 j = 0
711 i = start
712 while mops.Greater(end, i): # i < end
713 s = d.get(i)
714 #log('s %s', s)
715 if s is not None:
716 items2[j] = s
717 j += 1
718
719 i = mops.Add(i, mops.ONE) # i += 1
720
721 # TODO: return SparseArray
722 return value.BashArray(items2)
723
724 elif op_name == 'append': # a+=(x y)
725 strs = rd.PosBashArray()
726
727 # TODO: We can maintain the max index in the value.SparseArray(),
728 # so that it's O(1) to append rather than O(n)
729 # - Update on 'set' is O(1)
730 # - Update on 'unset' is potentially O(n)
731
732 if 0:
733 max_index = mops.MINUS_ONE # Note: this works for empty arrays
734 for i1 in d:
735 if mops.Greater(i1, max_index): # i1 > max_index
736 max_index = i1
737 else:
738 max_index = sp.max_index
739
740 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
741 for s in strs:
742 d[i2] = s
743 i2 = mops.Add(i2, mops.ONE) # i2 += 1
744
745 # sp.max_index += len(strs)
746 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
747 return value.Int(mops.ZERO)
748
749 else:
750 print('Invalid SparseArray operation %r' % op_name)
751 return value.Int(mops.ZERO)