OILS / builtin / func_misc.py View on Github | oilshell.org

754 lines, 429 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str)
9
10from core import error
11from core import num
12from core import state
13from core import ui
14from core import vm
15from data_lang import j8
16from frontend import match
17from frontend import typed_args
18from mycpp import mops
19from mycpp import mylib
20from mycpp.mylib import NewDict, iteritems, log, tagswitch
21from ysh import expr_eval
22from ysh import val_ops
23
24from typing import TYPE_CHECKING, Dict, List, cast
25if TYPE_CHECKING:
26 from osh import glob_
27 from osh import split
28
29_ = log
30
31
32class Len(vm._Callable):
33
34 def __init__(self):
35 # type: () -> None
36 pass
37
38 def Call(self, rd):
39 # type: (typed_args.Reader) -> value_t
40
41 x = rd.PosValue()
42 rd.Done()
43
44 UP_x = x
45 with tagswitch(x) as case:
46 if case(value_e.List):
47 x = cast(value.List, UP_x)
48 return num.ToBig(len(x.items))
49
50 elif case(value_e.Dict):
51 x = cast(value.Dict, UP_x)
52 return num.ToBig(len(x.d))
53
54 elif case(value_e.Str):
55 x = cast(value.Str, UP_x)
56 return num.ToBig(len(x.s))
57
58 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
59 rd.BlamePos())
60
61
62class Type(vm._Callable):
63
64 def __init__(self):
65 # type: () -> None
66 pass
67
68 def Call(self, rd):
69 # type: (typed_args.Reader) -> value_t
70
71 val = rd.PosValue()
72 rd.Done()
73
74 return value.Str(ui.ValType(val))
75
76
77class Repeat(vm._Callable):
78
79 def __init__(self):
80 # type: () -> None
81 pass
82
83 def Call(self, rd):
84 # type: (typed_args.Reader) -> value_t
85 return value.Null
86
87
88class Join(vm._Callable):
89 """Both free function join() and List->join() method."""
90
91 def __init__(self):
92 # type: () -> None
93 pass
94
95 def Call(self, rd):
96 # type: (typed_args.Reader) -> value_t
97
98 li = rd.PosList()
99 delim = rd.OptionalStr(default_='')
100 rd.Done()
101
102 strs = [] # type: List[str]
103 for i, el in enumerate(li):
104 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
105
106 return value.Str(delim.join(strs))
107
108
109class Maybe(vm._Callable):
110
111 def __init__(self):
112 # type: () -> None
113 pass
114
115 def Call(self, rd):
116 # type: (typed_args.Reader) -> value_t
117
118 val = rd.PosValue()
119 rd.Done()
120
121 if val == value.Null:
122 return value.List([])
123
124 s = val_ops.ToStr(
125 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
126 rd.LeftParenToken())
127 if len(s):
128 return value.List([val]) # use val to avoid needlessly copy
129
130 return value.List([])
131
132
133class Bool(vm._Callable):
134
135 def __init__(self):
136 # type: () -> None
137 pass
138
139 def Call(self, rd):
140 # type: (typed_args.Reader) -> value_t
141
142 val = rd.PosValue()
143 rd.Done()
144
145 return value.Bool(val_ops.ToBool(val))
146
147
148class Int(vm._Callable):
149
150 def __init__(self):
151 # type: () -> None
152 pass
153
154 def Call(self, rd):
155 # type: (typed_args.Reader) -> value_t
156
157 val = rd.PosValue()
158 rd.Done()
159
160 UP_val = val
161 with tagswitch(val) as case:
162 if case(value_e.Int):
163 return val
164
165 elif case(value_e.Bool):
166 val = cast(value.Bool, UP_val)
167 return value.Int(mops.FromBool(val.b))
168
169 elif case(value_e.Float):
170 val = cast(value.Float, UP_val)
171 return value.Int(mops.FromFloat(val.f))
172
173 elif case(value_e.Str):
174 val = cast(value.Str, UP_val)
175 if not match.LooksLikeInteger(val.s):
176 raise error.Expr('Cannot convert %s to Int' % val.s,
177 rd.BlamePos())
178
179 return value.Int(mops.FromStr(val.s))
180
181 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
182 rd.BlamePos())
183
184
185class Float(vm._Callable):
186
187 def __init__(self):
188 # type: () -> None
189 pass
190
191 def Call(self, rd):
192 # type: (typed_args.Reader) -> value_t
193
194 val = rd.PosValue()
195 rd.Done()
196
197 UP_val = val
198 with tagswitch(val) as case:
199 if case(value_e.Int):
200 val = cast(value.Int, UP_val)
201 return value.Float(mops.ToFloat(val.i))
202
203 elif case(value_e.Float):
204 return val
205
206 elif case(value_e.Str):
207 val = cast(value.Str, UP_val)
208 if not match.LooksLikeFloat(val.s):
209 raise error.Expr('Cannot convert %s to Float' % val.s,
210 rd.BlamePos())
211
212 return value.Float(float(val.s))
213
214 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
215 rd.BlamePos())
216
217
218class Str_(vm._Callable):
219
220 def __init__(self):
221 # type: () -> None
222 pass
223
224 def Call(self, rd):
225 # type: (typed_args.Reader) -> value_t
226
227 val = rd.PosValue()
228 rd.Done()
229
230 # TODO: Should we call Stringify here? That would handle Eggex.
231
232 UP_val = val
233 with tagswitch(val) as case:
234 if case(value_e.Int):
235 val = cast(value.Int, UP_val)
236 return value.Str(mops.ToStr(val.i))
237
238 elif case(value_e.Float):
239 val = cast(value.Float, UP_val)
240 return value.Str(str(val.f))
241
242 elif case(value_e.Str):
243 return val
244
245 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
246 rd.BlamePos())
247
248
249class List_(vm._Callable):
250
251 def __init__(self):
252 # type: () -> None
253 pass
254
255 def Call(self, rd):
256 # type: (typed_args.Reader) -> value_t
257
258 val = rd.PosValue()
259 rd.Done()
260
261 l = [] # type: List[value_t]
262 it = None # type: val_ops.Iterator
263 UP_val = val
264 with tagswitch(val) as case:
265 if case(value_e.List):
266 val = cast(value.List, UP_val)
267 it = val_ops.ListIterator(val)
268
269 elif case(value_e.Dict):
270 val = cast(value.Dict, UP_val)
271 it = val_ops.DictIterator(val)
272
273 elif case(value_e.Range):
274 val = cast(value.Range, UP_val)
275 it = val_ops.RangeIterator(val)
276
277 else:
278 raise error.TypeErr(val,
279 'list() expected Dict, List, or Range',
280 rd.BlamePos())
281
282 assert it is not None
283 while True:
284 first = it.FirstValue()
285 if first is None:
286 break
287 l.append(first)
288 it.Next()
289
290 return value.List(l)
291
292
293class Dict_(vm._Callable):
294
295 def __init__(self):
296 # type: () -> None
297 pass
298
299 def Call(self, rd):
300 # type: (typed_args.Reader) -> value_t
301
302 val = rd.PosValue()
303 rd.Done()
304
305 UP_val = val
306 with tagswitch(val) as case:
307 if case(value_e.Dict):
308 d = NewDict() # type: Dict[str, value_t]
309 val = cast(value.Dict, UP_val)
310 for k, v in iteritems(val.d):
311 d[k] = v
312
313 return value.Dict(d)
314
315 elif case(value_e.BashAssoc):
316 d = NewDict()
317 val = cast(value.BashAssoc, UP_val)
318 for k, s in iteritems(val.d):
319 d[k] = value.Str(s)
320
321 return value.Dict(d)
322
323 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
324 rd.BlamePos())
325
326
327class Runes(vm._Callable):
328
329 def __init__(self):
330 # type: () -> None
331 pass
332
333 def Call(self, rd):
334 # type: (typed_args.Reader) -> value_t
335 return value.Null
336
337
338class EncodeRunes(vm._Callable):
339
340 def __init__(self):
341 # type: () -> None
342 pass
343
344 def Call(self, rd):
345 # type: (typed_args.Reader) -> value_t
346 return value.Null
347
348
349class Bytes(vm._Callable):
350
351 def __init__(self):
352 # type: () -> None
353 pass
354
355 def Call(self, rd):
356 # type: (typed_args.Reader) -> value_t
357 return value.Null
358
359
360class EncodeBytes(vm._Callable):
361
362 def __init__(self):
363 # type: () -> None
364 pass
365
366 def Call(self, rd):
367 # type: (typed_args.Reader) -> value_t
368 return value.Null
369
370
371class Split(vm._Callable):
372
373 def __init__(self, splitter):
374 # type: (split.SplitContext) -> None
375 vm._Callable.__init__(self)
376 self.splitter = splitter
377
378 def Call(self, rd):
379 # type: (typed_args.Reader) -> value_t
380 s = rd.PosStr()
381
382 ifs = rd.OptionalStr()
383
384 rd.Done()
385
386 l = [
387 value.Str(elem)
388 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
389 ] # type: List[value_t]
390 return value.List(l)
391
392
393class Glob(vm._Callable):
394
395 def __init__(self, globber):
396 # type: (glob_.Globber) -> None
397 vm._Callable.__init__(self)
398 self.globber = globber
399
400 def Call(self, rd):
401 # type: (typed_args.Reader) -> value_t
402 s = rd.PosStr()
403 rd.Done()
404
405 out = [] # type: List[str]
406 self.globber._Glob(s, out)
407
408 l = [value.Str(elem) for elem in out] # type: List[value_t]
409 return value.List(l)
410
411
412class Shvar_get(vm._Callable):
413 """Look up with dynamic scope."""
414
415 def __init__(self, mem):
416 # type: (state.Mem) -> None
417 vm._Callable.__init__(self)
418 self.mem = mem
419
420 def Call(self, rd):
421 # type: (typed_args.Reader) -> value_t
422 name = rd.PosStr()
423 rd.Done()
424 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
425
426
427class GetVar(vm._Callable):
428 """Look up normal scoping rules."""
429
430 def __init__(self, mem):
431 # type: (state.Mem) -> None
432 vm._Callable.__init__(self)
433 self.mem = mem
434
435 def Call(self, rd):
436 # type: (typed_args.Reader) -> value_t
437 name = rd.PosStr()
438 rd.Done()
439 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
440
441
442class Assert(vm._Callable):
443
444 def __init__(self):
445 # type: () -> None
446 pass
447
448 def Call(self, rd):
449 # type: (typed_args.Reader) -> value_t
450
451 val = rd.PosValue()
452
453 msg = rd.OptionalStr(default_='')
454
455 rd.Done()
456
457 if not val_ops.ToBool(val):
458 raise error.AssertionErr(msg, rd.LeftParenToken())
459
460 return value.Null
461
462
463class EvalExpr(vm._Callable):
464
465 def __init__(self, expr_ev):
466 # type: (expr_eval.ExprEvaluator) -> None
467 self.expr_ev = expr_ev
468
469 def Call(self, rd):
470 # type: (typed_args.Reader) -> value_t
471 lazy = rd.PosExpr()
472 rd.Done()
473
474 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
475
476 return result
477
478
479class ToJson8(vm._Callable):
480
481 def __init__(self, is_j8):
482 # type: (bool) -> None
483 self.is_j8 = is_j8
484
485 def Call(self, rd):
486 # type: (typed_args.Reader) -> value_t
487
488 val = rd.PosValue()
489 space = mops.BigTruncate(rd.NamedInt('space', 0))
490 rd.Done()
491
492 # Convert from external JS-like API to internal API.
493 if space <= 0:
494 indent = -1
495 else:
496 indent = space
497
498 buf = mylib.BufWriter()
499 try:
500 if self.is_j8:
501 j8.PrintMessage(val, buf, indent)
502 else:
503 j8.PrintJsonMessage(val, buf, indent)
504 except error.Encode as e:
505 # status code 4 is special, for encode/decode errors.
506 raise error.Structured(4, e.Message(), rd.LeftParenToken())
507
508 return value.Str(buf.getvalue())
509
510
511class FromJson8(vm._Callable):
512
513 def __init__(self, is_j8):
514 # type: (bool) -> None
515 self.is_j8 = is_j8
516
517 def Call(self, rd):
518 # type: (typed_args.Reader) -> value_t
519
520 s = rd.PosStr()
521 rd.Done()
522
523 p = j8.Parser(s, self.is_j8)
524 try:
525 val = p.ParseValue()
526 except error.Decode as e:
527 # Right now I'm not exposing the original string, because that
528 # could lead to a memory leak in the _error Dict.
529 # The message quotes part of the string, and we could improve
530 # that. We could have a substring with context.
531 props = {
532 'start_pos': num.ToBig(e.start_pos),
533 'end_pos': num.ToBig(e.end_pos),
534 } # type: Dict[str, value_t]
535 # status code 4 is special, for encode/decode errors.
536 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
537
538 return val
539
540
541class BashArrayToSparse(vm._Callable):
542 """
543 value.BashArray -> value.SparseArray, for testing
544 """
545
546 def __init__(self):
547 # type: () -> None
548 pass
549
550 def Call(self, rd):
551 # type: (typed_args.Reader) -> value_t
552
553 strs = rd.PosBashArray()
554 rd.Done()
555
556 d = {} # type: Dict[mops.BigInt, str]
557 max_index = mops.MINUS_ONE # max index for empty array
558 for i, s in enumerate(strs):
559 if s is not None:
560 big_i = mops.IntWiden(i)
561 d[big_i] = s
562 if mops.Greater(big_i, max_index):
563 max_index = big_i
564
565 return value.SparseArray(d, max_index)
566
567
568class DictToSparse(vm._Callable):
569 """
570 value.Dict -> value.SparseArray, for testing
571 """
572
573 def __init__(self):
574 # type: () -> None
575 pass
576
577 def Call(self, rd):
578 # type: (typed_args.Reader) -> value_t
579
580 d = rd.PosDict()
581 rd.Done()
582
583 blame_tok = rd.LeftParenToken()
584
585 mydict = {} # type: Dict[mops.BigInt, str]
586 for k, v in iteritems(d):
587 i = mops.FromStr(k)
588 s = val_ops.ToStr(v, 'expected str', blame_tok)
589
590 mydict[i] = s
591
592 max_index = mops.MINUS_ONE # TODO:
593 return value.SparseArray(mydict, max_index)
594
595
596class SparseOp(vm._Callable):
597 """
598 All ops on value.SparseArray, for testing performance
599 """
600
601 def __init__(self):
602 # type: () -> None
603 pass
604
605 def Call(self, rd):
606 # type: (typed_args.Reader) -> value_t
607
608 sp = rd.PosSparseArray()
609 d = sp.d
610 #i = mops.BigTruncate(rd.PosInt())
611 op_name = rd.PosStr()
612
613 no_str = None # type: str
614
615 if op_name == 'len': # ${#a[@]}
616 rd.Done()
617 return num.ToBig(len(d))
618
619 elif op_name == 'get': # ${a[42]}
620 index = rd.PosInt()
621 rd.Done()
622
623 s = d.get(index)
624 if s is None:
625 return value.Null
626 else:
627 return value.Str(s)
628
629 elif op_name == 'set': # a[42]=foo
630 index = rd.PosInt()
631 s = rd.PosStr()
632 rd.Done()
633
634 d[index] = s
635
636 if mops.Greater(index, sp.max_index):
637 sp.max_index = index
638
639 return value.Int(mops.ZERO)
640
641 elif op_name == 'unset': # unset 'a[1]'
642 index = rd.PosInt()
643 rd.Done()
644
645 mylib.dict_erase(d, index)
646
647 max_index = mops.MINUS_ONE # Note: this works if d is not empty
648 for i1 in d:
649 if mops.Greater(i1, max_index): # i1 > max_index
650 max_index = i1
651 sp.max_index = max_index
652
653 return value.Int(mops.ZERO)
654
655 elif op_name == 'subst': # "${a[@]}"
656 # Algorithm to expand a Dict[BigInt, Str]
657 #
658 # 1. Copy the integer keys into a new List
659 # 2. Sort them in numeric order
660 # 3. Create a List[str] that's the same size as the keys
661 # 4. Loop through sorted keys, look up value, and populate list
662 #
663 # There is another possible algorithm:
664 #
665 # 1. Copy the VALUES into a new list
666 # 2. Somehow sort them by the CORRESPONDING key, which depends on
667 # Slab<> POSITION. I think this does not fit within the
668 # std::sort() model. I think we would have to write a little custom
669 # sort algorithm.
670
671 keys = d.keys()
672 mylib.BigIntSort(keys)
673 # Pre-allocate
674 items = [no_str] * len(d) # type: List[str]
675 j = 0
676 for i in keys:
677 s = d.get(i)
678 assert s is not None
679 items[j] = s
680 j += 1
681 return value.BashArray(items)
682
683 elif op_name == 'keys': # "${!a[@]}"
684 keys = d.keys()
685 mylib.BigIntSort(keys)
686 items = [mops.ToStr(k) for k in keys]
687
688 # TODO: return SparseArray
689 return value.BashArray(items)
690
691 elif op_name == 'slice': # "${a[@]:0:5}"
692 start = rd.PosInt()
693 end = rd.PosInt()
694 rd.Done()
695
696 n = mops.BigTruncate(mops.Sub(end, start))
697 #log('start %d - end %d', start.i, end.i)
698
699 # Pre-allocate
700 items2 = [no_str] * n # type: List[str]
701
702 # Iterate from start to end. Note that this algorithm is
703 # theoretically slower than bash in the case where the array is
704 # sparse (in the part selected by the slice)
705 #
706 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
707 # elements, OSH will iterate through 999 integers and do 999 dict
708 # lookups, while bash will follow 3 pointers.
709 #
710 # However, in practice, I think iterating through integers is
711 # cheap.
712
713 j = 0
714 i = start
715 while mops.Greater(end, i): # i < end
716 s = d.get(i)
717 #log('s %s', s)
718 if s is not None:
719 items2[j] = s
720 j += 1
721
722 i = mops.Add(i, mops.ONE) # i += 1
723
724 # TODO: return SparseArray
725 return value.BashArray(items2)
726
727 elif op_name == 'append': # a+=(x y)
728 strs = rd.PosBashArray()
729
730 # TODO: We can maintain the max index in the value.SparseArray(),
731 # so that it's O(1) to append rather than O(n)
732 # - Update on 'set' is O(1)
733 # - Update on 'unset' is potentially O(n)
734
735 if 0:
736 max_index = mops.MINUS_ONE # Note: this works for empty arrays
737 for i1 in d:
738 if mops.Greater(i1, max_index): # i1 > max_index
739 max_index = i1
740 else:
741 max_index = sp.max_index
742
743 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
744 for s in strs:
745 d[i2] = s
746 i2 = mops.Add(i2, mops.ONE) # i2 += 1
747
748 # sp.max_index += len(strs)
749 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
750 return value.Int(mops.ZERO)
751
752 else:
753 print('Invalid SparseArray operation %r' % op_name)
754 return value.Int(mops.ZERO)