OILS / builtin / func_misc.py View on Github | oilshell.org

758 lines, 432 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str)
9
10from core import error
11from core import num
12from core import state
13from core import ui
14from core import vm
15from data_lang import j8
16from frontend import match
17from frontend import typed_args
18from mycpp import mops
19from mycpp import mylib
20from mycpp.mylib import NewDict, iteritems, log, tagswitch
21from ysh import expr_eval
22from ysh import val_ops
23
24from typing import TYPE_CHECKING, Dict, List, cast
25if TYPE_CHECKING:
26 from osh import glob_
27 from osh import split
28
29_ = log
30
31
32class Len(vm._Callable):
33
34 def __init__(self):
35 # type: () -> None
36 pass
37
38 def Call(self, rd):
39 # type: (typed_args.Reader) -> value_t
40
41 x = rd.PosValue()
42 rd.Done()
43
44 UP_x = x
45 with tagswitch(x) as case:
46 if case(value_e.List):
47 x = cast(value.List, UP_x)
48 return num.ToBig(len(x.items))
49
50 elif case(value_e.Dict):
51 x = cast(value.Dict, UP_x)
52 return num.ToBig(len(x.d))
53
54 elif case(value_e.Str):
55 x = cast(value.Str, UP_x)
56 return num.ToBig(len(x.s))
57
58 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
59 rd.BlamePos())
60
61
62class Type(vm._Callable):
63
64 def __init__(self):
65 # type: () -> None
66 pass
67
68 def Call(self, rd):
69 # type: (typed_args.Reader) -> value_t
70
71 val = rd.PosValue()
72 rd.Done()
73
74 return value.Str(ui.ValType(val))
75
76
77class Join(vm._Callable):
78 """Both free function join() and List->join() method."""
79
80 def __init__(self):
81 # type: () -> None
82 pass
83
84 def Call(self, rd):
85 # type: (typed_args.Reader) -> value_t
86
87 li = rd.PosList()
88 delim = rd.OptionalStr(default_='')
89 rd.Done()
90
91 strs = [] # type: List[str]
92 for i, el in enumerate(li):
93 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
94
95 return value.Str(delim.join(strs))
96
97
98class Maybe(vm._Callable):
99
100 def __init__(self):
101 # type: () -> None
102 pass
103
104 def Call(self, rd):
105 # type: (typed_args.Reader) -> value_t
106
107 val = rd.PosValue()
108 rd.Done()
109
110 if val == value.Null:
111 return value.List([])
112
113 s = val_ops.ToStr(
114 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
115 rd.LeftParenToken())
116 if len(s):
117 return value.List([val]) # use val to avoid needlessly copy
118
119 return value.List([])
120
121
122class Bool(vm._Callable):
123
124 def __init__(self):
125 # type: () -> None
126 pass
127
128 def Call(self, rd):
129 # type: (typed_args.Reader) -> value_t
130
131 val = rd.PosValue()
132 rd.Done()
133
134 return value.Bool(val_ops.ToBool(val))
135
136
137class Int(vm._Callable):
138
139 def __init__(self):
140 # type: () -> None
141 pass
142
143 def Call(self, rd):
144 # type: (typed_args.Reader) -> value_t
145
146 val = rd.PosValue()
147 rd.Done()
148
149 UP_val = val
150 with tagswitch(val) as case:
151 if case(value_e.Int):
152 return val
153
154 elif case(value_e.Bool):
155 val = cast(value.Bool, UP_val)
156 return value.Int(mops.FromBool(val.b))
157
158 elif case(value_e.Float):
159 val = cast(value.Float, UP_val)
160 return value.Int(mops.FromFloat(val.f))
161
162 elif case(value_e.Str):
163 val = cast(value.Str, UP_val)
164 if not match.LooksLikeInteger(val.s):
165 raise error.Expr('Cannot convert %s to Int' % val.s,
166 rd.BlamePos())
167
168 return value.Int(mops.FromStr(val.s))
169
170 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
171 rd.BlamePos())
172
173
174class Float(vm._Callable):
175
176 def __init__(self):
177 # type: () -> None
178 pass
179
180 def Call(self, rd):
181 # type: (typed_args.Reader) -> value_t
182
183 val = rd.PosValue()
184 rd.Done()
185
186 UP_val = val
187 with tagswitch(val) as case:
188 if case(value_e.Int):
189 val = cast(value.Int, UP_val)
190 return value.Float(mops.ToFloat(val.i))
191
192 elif case(value_e.Float):
193 return val
194
195 elif case(value_e.Str):
196 val = cast(value.Str, UP_val)
197 if not match.LooksLikeFloat(val.s):
198 raise error.Expr('Cannot convert %s to Float' % val.s,
199 rd.BlamePos())
200
201 return value.Float(float(val.s))
202
203 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
204 rd.BlamePos())
205
206
207class Str_(vm._Callable):
208
209 def __init__(self):
210 # type: () -> None
211 pass
212
213 def Call(self, rd):
214 # type: (typed_args.Reader) -> value_t
215
216 val = rd.PosValue()
217 rd.Done()
218
219 # TODO: Should we call Stringify here? That would handle Eggex.
220
221 UP_val = val
222 with tagswitch(val) as case:
223 if case(value_e.Int):
224 val = cast(value.Int, UP_val)
225 return value.Str(mops.ToStr(val.i))
226
227 elif case(value_e.Float):
228 val = cast(value.Float, UP_val)
229 return value.Str(str(val.f))
230
231 elif case(value_e.Str):
232 return val
233
234 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
235 rd.BlamePos())
236
237
238class List_(vm._Callable):
239
240 def __init__(self):
241 # type: () -> None
242 pass
243
244 def Call(self, rd):
245 # type: (typed_args.Reader) -> value_t
246
247 val = rd.PosValue()
248 rd.Done()
249
250 l = [] # type: List[value_t]
251 it = None # type: val_ops.Iterator
252 UP_val = val
253 with tagswitch(val) as case:
254 if case(value_e.List):
255 val = cast(value.List, UP_val)
256 it = val_ops.ListIterator(val)
257
258 elif case(value_e.Dict):
259 val = cast(value.Dict, UP_val)
260 it = val_ops.DictIterator(val)
261
262 elif case(value_e.Range):
263 val = cast(value.Range, UP_val)
264 it = val_ops.RangeIterator(val)
265
266 else:
267 raise error.TypeErr(val,
268 'list() expected Dict, List, or Range',
269 rd.BlamePos())
270
271 assert it is not None
272 while True:
273 first = it.FirstValue()
274 if first is None:
275 break
276 l.append(first)
277 it.Next()
278
279 return value.List(l)
280
281
282class Dict_(vm._Callable):
283
284 def __init__(self):
285 # type: () -> None
286 pass
287
288 def Call(self, rd):
289 # type: (typed_args.Reader) -> value_t
290
291 val = rd.PosValue()
292 rd.Done()
293
294 UP_val = val
295 with tagswitch(val) as case:
296 if case(value_e.Dict):
297 d = NewDict() # type: Dict[str, value_t]
298 val = cast(value.Dict, UP_val)
299 for k, v in iteritems(val.d):
300 d[k] = v
301
302 return value.Dict(d)
303
304 elif case(value_e.BashAssoc):
305 d = NewDict()
306 val = cast(value.BashAssoc, UP_val)
307 for k, s in iteritems(val.d):
308 d[k] = value.Str(s)
309
310 return value.Dict(d)
311
312 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
313 rd.BlamePos())
314
315
316class Runes(vm._Callable):
317
318 def __init__(self):
319 # type: () -> None
320 pass
321
322 def Call(self, rd):
323 # type: (typed_args.Reader) -> value_t
324 return value.Null
325
326
327class EncodeRunes(vm._Callable):
328
329 def __init__(self):
330 # type: () -> None
331 pass
332
333 def Call(self, rd):
334 # type: (typed_args.Reader) -> value_t
335 return value.Null
336
337
338class Bytes(vm._Callable):
339
340 def __init__(self):
341 # type: () -> None
342 pass
343
344 def Call(self, rd):
345 # type: (typed_args.Reader) -> value_t
346 return value.Null
347
348
349class EncodeBytes(vm._Callable):
350
351 def __init__(self):
352 # type: () -> None
353 pass
354
355 def Call(self, rd):
356 # type: (typed_args.Reader) -> value_t
357 return value.Null
358
359
360class Split(vm._Callable):
361
362 def __init__(self, splitter):
363 # type: (split.SplitContext) -> None
364 vm._Callable.__init__(self)
365 self.splitter = splitter
366
367 def Call(self, rd):
368 # type: (typed_args.Reader) -> value_t
369 s = rd.PosStr()
370
371 ifs = rd.OptionalStr()
372
373 rd.Done()
374
375 l = [
376 value.Str(elem)
377 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
378 ] # type: List[value_t]
379 return value.List(l)
380
381
382class FloatsEqual(vm._Callable):
383
384 def __init__(self):
385 # type: () -> None
386 pass
387
388 def Call(self, rd):
389 # type: (typed_args.Reader) -> value_t
390 left = rd.PosFloat()
391 right = rd.PosFloat()
392 rd.Done()
393
394 return value.Bool(left == right)
395
396
397class Glob(vm._Callable):
398
399 def __init__(self, globber):
400 # type: (glob_.Globber) -> None
401 vm._Callable.__init__(self)
402 self.globber = globber
403
404 def Call(self, rd):
405 # type: (typed_args.Reader) -> value_t
406 s = rd.PosStr()
407 rd.Done()
408
409 out = [] # type: List[str]
410 self.globber._Glob(s, out)
411
412 l = [value.Str(elem) for elem in out] # type: List[value_t]
413 return value.List(l)
414
415
416class Shvar_get(vm._Callable):
417 """Look up with dynamic scope."""
418
419 def __init__(self, mem):
420 # type: (state.Mem) -> None
421 vm._Callable.__init__(self)
422 self.mem = mem
423
424 def Call(self, rd):
425 # type: (typed_args.Reader) -> value_t
426 name = rd.PosStr()
427 rd.Done()
428 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
429
430
431class GetVar(vm._Callable):
432 """Look up normal scoping rules."""
433
434 def __init__(self, mem):
435 # type: (state.Mem) -> None
436 vm._Callable.__init__(self)
437 self.mem = mem
438
439 def Call(self, rd):
440 # type: (typed_args.Reader) -> value_t
441 name = rd.PosStr()
442 rd.Done()
443 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
444
445
446class Assert(vm._Callable):
447
448 def __init__(self):
449 # type: () -> None
450 pass
451
452 def Call(self, rd):
453 # type: (typed_args.Reader) -> value_t
454
455 val = rd.PosValue()
456
457 msg = rd.OptionalStr(default_='')
458
459 rd.Done()
460
461 if not val_ops.ToBool(val):
462 raise error.AssertionErr(msg, rd.LeftParenToken())
463
464 return value.Null
465
466
467class EvalExpr(vm._Callable):
468
469 def __init__(self, expr_ev):
470 # type: (expr_eval.ExprEvaluator) -> None
471 self.expr_ev = expr_ev
472
473 def Call(self, rd):
474 # type: (typed_args.Reader) -> value_t
475 lazy = rd.PosExpr()
476 rd.Done()
477
478 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
479
480 return result
481
482
483class ToJson8(vm._Callable):
484
485 def __init__(self, is_j8):
486 # type: (bool) -> None
487 self.is_j8 = is_j8
488
489 def Call(self, rd):
490 # type: (typed_args.Reader) -> value_t
491
492 val = rd.PosValue()
493 space = mops.BigTruncate(rd.NamedInt('space', 0))
494 rd.Done()
495
496 # Convert from external JS-like API to internal API.
497 if space <= 0:
498 indent = -1
499 else:
500 indent = space
501
502 buf = mylib.BufWriter()
503 try:
504 if self.is_j8:
505 j8.PrintMessage(val, buf, indent)
506 else:
507 j8.PrintJsonMessage(val, buf, indent)
508 except error.Encode as e:
509 # status code 4 is special, for encode/decode errors.
510 raise error.Structured(4, e.Message(), rd.LeftParenToken())
511
512 return value.Str(buf.getvalue())
513
514
515class FromJson8(vm._Callable):
516
517 def __init__(self, is_j8):
518 # type: (bool) -> None
519 self.is_j8 = is_j8
520
521 def Call(self, rd):
522 # type: (typed_args.Reader) -> value_t
523
524 s = rd.PosStr()
525 rd.Done()
526
527 p = j8.Parser(s, self.is_j8)
528 try:
529 val = p.ParseValue()
530 except error.Decode as e:
531 # Right now I'm not exposing the original string, because that
532 # could lead to a memory leak in the _error Dict.
533 # The message quotes part of the string, and we could improve
534 # that. We could have a substring with context.
535 props = {
536 'start_pos': num.ToBig(e.start_pos),
537 'end_pos': num.ToBig(e.end_pos),
538 } # type: Dict[str, value_t]
539 # status code 4 is special, for encode/decode errors.
540 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
541
542 return val
543
544
545class BashArrayToSparse(vm._Callable):
546 """
547 value.BashArray -> value.SparseArray, for testing
548 """
549
550 def __init__(self):
551 # type: () -> None
552 pass
553
554 def Call(self, rd):
555 # type: (typed_args.Reader) -> value_t
556
557 strs = rd.PosBashArray()
558 rd.Done()
559
560 d = {} # type: Dict[mops.BigInt, str]
561 max_index = mops.MINUS_ONE # max index for empty array
562 for i, s in enumerate(strs):
563 if s is not None:
564 big_i = mops.IntWiden(i)
565 d[big_i] = s
566 if mops.Greater(big_i, max_index):
567 max_index = big_i
568
569 return value.SparseArray(d, max_index)
570
571
572class DictToSparse(vm._Callable):
573 """
574 value.Dict -> value.SparseArray, for testing
575 """
576
577 def __init__(self):
578 # type: () -> None
579 pass
580
581 def Call(self, rd):
582 # type: (typed_args.Reader) -> value_t
583
584 d = rd.PosDict()
585 rd.Done()
586
587 blame_tok = rd.LeftParenToken()
588
589 mydict = {} # type: Dict[mops.BigInt, str]
590 for k, v in iteritems(d):
591 i = mops.FromStr(k)
592 s = val_ops.ToStr(v, 'expected str', blame_tok)
593
594 mydict[i] = s
595
596 max_index = mops.MINUS_ONE # TODO:
597 return value.SparseArray(mydict, max_index)
598
599
600class SparseOp(vm._Callable):
601 """
602 All ops on value.SparseArray, for testing performance
603 """
604
605 def __init__(self):
606 # type: () -> None
607 pass
608
609 def Call(self, rd):
610 # type: (typed_args.Reader) -> value_t
611
612 sp = rd.PosSparseArray()
613 d = sp.d
614 #i = mops.BigTruncate(rd.PosInt())
615 op_name = rd.PosStr()
616
617 no_str = None # type: str
618
619 if op_name == 'len': # ${#a[@]}
620 rd.Done()
621 return num.ToBig(len(d))
622
623 elif op_name == 'get': # ${a[42]}
624 index = rd.PosInt()
625 rd.Done()
626
627 s = d.get(index)
628 if s is None:
629 return value.Null
630 else:
631 return value.Str(s)
632
633 elif op_name == 'set': # a[42]=foo
634 index = rd.PosInt()
635 s = rd.PosStr()
636 rd.Done()
637
638 d[index] = s
639
640 if mops.Greater(index, sp.max_index):
641 sp.max_index = index
642
643 return value.Int(mops.ZERO)
644
645 elif op_name == 'unset': # unset 'a[1]'
646 index = rd.PosInt()
647 rd.Done()
648
649 mylib.dict_erase(d, index)
650
651 max_index = mops.MINUS_ONE # Note: this works if d is not empty
652 for i1 in d:
653 if mops.Greater(i1, max_index): # i1 > max_index
654 max_index = i1
655 sp.max_index = max_index
656
657 return value.Int(mops.ZERO)
658
659 elif op_name == 'subst': # "${a[@]}"
660 # Algorithm to expand a Dict[BigInt, Str]
661 #
662 # 1. Copy the integer keys into a new List
663 # 2. Sort them in numeric order
664 # 3. Create a List[str] that's the same size as the keys
665 # 4. Loop through sorted keys, look up value, and populate list
666 #
667 # There is another possible algorithm:
668 #
669 # 1. Copy the VALUES into a new list
670 # 2. Somehow sort them by the CORRESPONDING key, which depends on
671 # Slab<> POSITION. I think this does not fit within the
672 # std::sort() model. I think we would have to write a little custom
673 # sort algorithm.
674
675 keys = d.keys()
676 mylib.BigIntSort(keys)
677 # Pre-allocate
678 items = [no_str] * len(d) # type: List[str]
679 j = 0
680 for i in keys:
681 s = d.get(i)
682 assert s is not None
683 items[j] = s
684 j += 1
685 return value.BashArray(items)
686
687 elif op_name == 'keys': # "${!a[@]}"
688 keys = d.keys()
689 mylib.BigIntSort(keys)
690 items = [mops.ToStr(k) for k in keys]
691
692 # TODO: return SparseArray
693 return value.BashArray(items)
694
695 elif op_name == 'slice': # "${a[@]:0:5}"
696 start = rd.PosInt()
697 end = rd.PosInt()
698 rd.Done()
699
700 n = mops.BigTruncate(mops.Sub(end, start))
701 #log('start %d - end %d', start.i, end.i)
702
703 # Pre-allocate
704 items2 = [no_str] * n # type: List[str]
705
706 # Iterate from start to end. Note that this algorithm is
707 # theoretically slower than bash in the case where the array is
708 # sparse (in the part selected by the slice)
709 #
710 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
711 # elements, OSH will iterate through 999 integers and do 999 dict
712 # lookups, while bash will follow 3 pointers.
713 #
714 # However, in practice, I think iterating through integers is
715 # cheap.
716
717 j = 0
718 i = start
719 while mops.Greater(end, i): # i < end
720 s = d.get(i)
721 #log('s %s', s)
722 if s is not None:
723 items2[j] = s
724 j += 1
725
726 i = mops.Add(i, mops.ONE) # i += 1
727
728 # TODO: return SparseArray
729 return value.BashArray(items2)
730
731 elif op_name == 'append': # a+=(x y)
732 strs = rd.PosBashArray()
733
734 # TODO: We can maintain the max index in the value.SparseArray(),
735 # so that it's O(1) to append rather than O(n)
736 # - Update on 'set' is O(1)
737 # - Update on 'unset' is potentially O(n)
738
739 if 0:
740 max_index = mops.MINUS_ONE # Note: this works for empty arrays
741 for i1 in d:
742 if mops.Greater(i1, max_index): # i1 > max_index
743 max_index = i1
744 else:
745 max_index = sp.max_index
746
747 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
748 for s in strs:
749 d[i2] = s
750 i2 = mops.Add(i2, mops.ONE) # i2 += 1
751
752 # sp.max_index += len(strs)
753 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
754 return value.Int(mops.ZERO)
755
756 else:
757 print('Invalid SparseArray operation %r' % op_name)
758 return value.Int(mops.ZERO)