OILS / builtin / func_misc.py View on Github | oilshell.org

709 lines, 408 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str)
9
10from core import error
11from core import num
12from core import state
13from core import ui
14from core import vm
15from data_lang import j8
16from frontend import match
17from frontend import typed_args
18from mycpp import mops
19from mycpp import mylib
20from mycpp.mylib import NewDict, iteritems, log, tagswitch
21from ysh import expr_eval
22from ysh import val_ops
23
24from typing import TYPE_CHECKING, Dict, List, cast
25if TYPE_CHECKING:
26 from osh import glob_
27 from osh import split
28
29_ = log
30
31
32class Len(vm._Callable):
33
34 def __init__(self):
35 # type: () -> None
36 pass
37
38 def Call(self, rd):
39 # type: (typed_args.Reader) -> value_t
40
41 x = rd.PosValue()
42 rd.Done()
43
44 UP_x = x
45 with tagswitch(x) as case:
46 if case(value_e.List):
47 x = cast(value.List, UP_x)
48 return num.ToBig(len(x.items))
49
50 elif case(value_e.Dict):
51 x = cast(value.Dict, UP_x)
52 return num.ToBig(len(x.d))
53
54 elif case(value_e.Str):
55 x = cast(value.Str, UP_x)
56 return num.ToBig(len(x.s))
57
58 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
59 rd.BlamePos())
60
61
62class Type(vm._Callable):
63
64 def __init__(self):
65 # type: () -> None
66 pass
67
68 def Call(self, rd):
69 # type: (typed_args.Reader) -> value_t
70
71 val = rd.PosValue()
72 rd.Done()
73
74 return value.Str(ui.ValType(val))
75
76
77class Join(vm._Callable):
78 """Both free function join() and List->join() method."""
79
80 def __init__(self):
81 # type: () -> None
82 pass
83
84 def Call(self, rd):
85 # type: (typed_args.Reader) -> value_t
86
87 li = rd.PosList()
88 delim = rd.OptionalStr(default_='')
89 rd.Done()
90
91 strs = [] # type: List[str]
92 for i, el in enumerate(li):
93 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
94
95 return value.Str(delim.join(strs))
96
97
98class Maybe(vm._Callable):
99
100 def __init__(self):
101 # type: () -> None
102 pass
103
104 def Call(self, rd):
105 # type: (typed_args.Reader) -> value_t
106
107 val = rd.PosValue()
108 rd.Done()
109
110 if val == value.Null:
111 return value.List([])
112
113 s = val_ops.ToStr(
114 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
115 rd.LeftParenToken())
116 if len(s):
117 return value.List([val]) # use val to avoid needlessly copy
118
119 return value.List([])
120
121
122class Bool(vm._Callable):
123
124 def __init__(self):
125 # type: () -> None
126 pass
127
128 def Call(self, rd):
129 # type: (typed_args.Reader) -> value_t
130
131 val = rd.PosValue()
132 rd.Done()
133
134 return value.Bool(val_ops.ToBool(val))
135
136
137class Int(vm._Callable):
138
139 def __init__(self):
140 # type: () -> None
141 pass
142
143 def Call(self, rd):
144 # type: (typed_args.Reader) -> value_t
145
146 val = rd.PosValue()
147 rd.Done()
148
149 UP_val = val
150 with tagswitch(val) as case:
151 if case(value_e.Int):
152 return val
153
154 elif case(value_e.Bool):
155 val = cast(value.Bool, UP_val)
156 return value.Int(mops.FromBool(val.b))
157
158 elif case(value_e.Float):
159 val = cast(value.Float, UP_val)
160 return value.Int(mops.FromFloat(val.f))
161
162 elif case(value_e.Str):
163 val = cast(value.Str, UP_val)
164 if not match.LooksLikeInteger(val.s):
165 raise error.Expr('Cannot convert %s to Int' % val.s,
166 rd.BlamePos())
167
168 return value.Int(mops.FromStr(val.s))
169
170 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
171 rd.BlamePos())
172
173
174class Float(vm._Callable):
175
176 def __init__(self):
177 # type: () -> None
178 pass
179
180 def Call(self, rd):
181 # type: (typed_args.Reader) -> value_t
182
183 val = rd.PosValue()
184 rd.Done()
185
186 UP_val = val
187 with tagswitch(val) as case:
188 if case(value_e.Int):
189 val = cast(value.Int, UP_val)
190 return value.Float(mops.ToFloat(val.i))
191
192 elif case(value_e.Float):
193 return val
194
195 elif case(value_e.Str):
196 val = cast(value.Str, UP_val)
197 if not match.LooksLikeFloat(val.s):
198 raise error.Expr('Cannot convert %s to Float' % val.s,
199 rd.BlamePos())
200
201 return value.Float(float(val.s))
202
203 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
204 rd.BlamePos())
205
206
207class Str_(vm._Callable):
208
209 def __init__(self):
210 # type: () -> None
211 pass
212
213 def Call(self, rd):
214 # type: (typed_args.Reader) -> value_t
215
216 val = rd.PosValue()
217 rd.Done()
218
219 # TODO: Should we call Stringify here? That would handle Eggex.
220
221 UP_val = val
222 with tagswitch(val) as case:
223 if case(value_e.Int):
224 val = cast(value.Int, UP_val)
225 return value.Str(mops.ToStr(val.i))
226
227 elif case(value_e.Float):
228 val = cast(value.Float, UP_val)
229 return value.Str(str(val.f))
230
231 elif case(value_e.Str):
232 return val
233
234 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
235 rd.BlamePos())
236
237
238class List_(vm._Callable):
239
240 def __init__(self):
241 # type: () -> None
242 pass
243
244 def Call(self, rd):
245 # type: (typed_args.Reader) -> value_t
246
247 val = rd.PosValue()
248 rd.Done()
249
250 l = [] # type: List[value_t]
251 it = None # type: val_ops.Iterator
252 UP_val = val
253 with tagswitch(val) as case:
254 if case(value_e.List):
255 val = cast(value.List, UP_val)
256 it = val_ops.ListIterator(val)
257
258 elif case(value_e.Dict):
259 val = cast(value.Dict, UP_val)
260 it = val_ops.DictIterator(val)
261
262 elif case(value_e.Range):
263 val = cast(value.Range, UP_val)
264 it = val_ops.RangeIterator(val)
265
266 else:
267 raise error.TypeErr(val,
268 'list() expected Dict, List, or Range',
269 rd.BlamePos())
270
271 assert it is not None
272 while True:
273 first = it.FirstValue()
274 if first is None:
275 break
276 l.append(first)
277 it.Next()
278
279 return value.List(l)
280
281
282class Dict_(vm._Callable):
283
284 def __init__(self):
285 # type: () -> None
286 pass
287
288 def Call(self, rd):
289 # type: (typed_args.Reader) -> value_t
290
291 val = rd.PosValue()
292 rd.Done()
293
294 UP_val = val
295 with tagswitch(val) as case:
296 if case(value_e.Dict):
297 d = NewDict() # type: Dict[str, value_t]
298 val = cast(value.Dict, UP_val)
299 for k, v in iteritems(val.d):
300 d[k] = v
301
302 return value.Dict(d)
303
304 elif case(value_e.BashAssoc):
305 d = NewDict()
306 val = cast(value.BashAssoc, UP_val)
307 for k, s in iteritems(val.d):
308 d[k] = value.Str(s)
309
310 return value.Dict(d)
311
312 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
313 rd.BlamePos())
314
315
316class Runes(vm._Callable):
317
318 def __init__(self):
319 # type: () -> None
320 pass
321
322 def Call(self, rd):
323 # type: (typed_args.Reader) -> value_t
324 return value.Null
325
326
327class EncodeRunes(vm._Callable):
328
329 def __init__(self):
330 # type: () -> None
331 pass
332
333 def Call(self, rd):
334 # type: (typed_args.Reader) -> value_t
335 return value.Null
336
337
338class Bytes(vm._Callable):
339
340 def __init__(self):
341 # type: () -> None
342 pass
343
344 def Call(self, rd):
345 # type: (typed_args.Reader) -> value_t
346 return value.Null
347
348
349class EncodeBytes(vm._Callable):
350
351 def __init__(self):
352 # type: () -> None
353 pass
354
355 def Call(self, rd):
356 # type: (typed_args.Reader) -> value_t
357 return value.Null
358
359
360class Split(vm._Callable):
361
362 def __init__(self, splitter):
363 # type: (split.SplitContext) -> None
364 vm._Callable.__init__(self)
365 self.splitter = splitter
366
367 def Call(self, rd):
368 # type: (typed_args.Reader) -> value_t
369 s = rd.PosStr()
370
371 ifs = rd.OptionalStr()
372
373 rd.Done()
374
375 l = [
376 value.Str(elem)
377 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
378 ] # type: List[value_t]
379 return value.List(l)
380
381
382class FloatsEqual(vm._Callable):
383
384 def __init__(self):
385 # type: () -> None
386 pass
387
388 def Call(self, rd):
389 # type: (typed_args.Reader) -> value_t
390 left = rd.PosFloat()
391 right = rd.PosFloat()
392 rd.Done()
393
394 return value.Bool(left == right)
395
396
397class Glob(vm._Callable):
398
399 def __init__(self, globber):
400 # type: (glob_.Globber) -> None
401 vm._Callable.__init__(self)
402 self.globber = globber
403
404 def Call(self, rd):
405 # type: (typed_args.Reader) -> value_t
406 s = rd.PosStr()
407 rd.Done()
408
409 out = [] # type: List[str]
410 self.globber._Glob(s, out)
411
412 l = [value.Str(elem) for elem in out] # type: List[value_t]
413 return value.List(l)
414
415
416class Shvar_get(vm._Callable):
417 """Look up with dynamic scope."""
418
419 def __init__(self, mem):
420 # type: (state.Mem) -> None
421 vm._Callable.__init__(self)
422 self.mem = mem
423
424 def Call(self, rd):
425 # type: (typed_args.Reader) -> value_t
426 name = rd.PosStr()
427 rd.Done()
428 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
429
430
431class GetVar(vm._Callable):
432 """Look up normal scoping rules."""
433
434 def __init__(self, mem):
435 # type: (state.Mem) -> None
436 vm._Callable.__init__(self)
437 self.mem = mem
438
439 def Call(self, rd):
440 # type: (typed_args.Reader) -> value_t
441 name = rd.PosStr()
442 rd.Done()
443 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
444
445
446class EvalExpr(vm._Callable):
447
448 def __init__(self, expr_ev):
449 # type: (expr_eval.ExprEvaluator) -> None
450 self.expr_ev = expr_ev
451
452 def Call(self, rd):
453 # type: (typed_args.Reader) -> value_t
454 lazy = rd.PosExpr()
455 rd.Done()
456
457 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
458
459 return result
460
461
462class ToJson8(vm._Callable):
463
464 def __init__(self, is_j8):
465 # type: (bool) -> None
466 self.is_j8 = is_j8
467
468 def Call(self, rd):
469 # type: (typed_args.Reader) -> value_t
470
471 val = rd.PosValue()
472 space = mops.BigTruncate(rd.NamedInt('space', 0))
473 rd.Done()
474
475 # Convert from external JS-like API to internal API.
476 if space <= 0:
477 indent = -1
478 else:
479 indent = space
480
481 buf = mylib.BufWriter()
482 try:
483 if self.is_j8:
484 j8.PrintMessage(val, buf, indent)
485 else:
486 j8.PrintJsonMessage(val, buf, indent)
487 except error.Encode as e:
488 # status code 4 is special, for encode/decode errors.
489 raise error.Structured(4, e.Message(), rd.LeftParenToken())
490
491 return value.Str(buf.getvalue())
492
493
494class FromJson8(vm._Callable):
495
496 def __init__(self, is_j8):
497 # type: (bool) -> None
498 self.is_j8 = is_j8
499
500 def Call(self, rd):
501 # type: (typed_args.Reader) -> value_t
502
503 s = rd.PosStr()
504 rd.Done()
505
506 p = j8.Parser(s, self.is_j8)
507 try:
508 val = p.ParseValue()
509 except error.Decode as e:
510 # Right now I'm not exposing the original string, because that
511 # could lead to a memory leak in the _error Dict.
512 # The message quotes part of the string, and we could improve
513 # that. We could have a substring with context.
514 props = {
515 'start_pos': num.ToBig(e.start_pos),
516 'end_pos': num.ToBig(e.end_pos),
517 } # type: Dict[str, value_t]
518 # status code 4 is special, for encode/decode errors.
519 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
520
521 return val
522
523
524class BashArrayToSparse(vm._Callable):
525 """
526 value.BashArray -> value.SparseArray, for testing
527 """
528
529 def __init__(self):
530 # type: () -> None
531 pass
532
533 def Call(self, rd):
534 # type: (typed_args.Reader) -> value_t
535
536 strs = rd.PosBashArray()
537 rd.Done()
538
539 d = {} # type: Dict[mops.BigInt, str]
540 max_index = mops.MINUS_ONE # max index for empty array
541 for i, s in enumerate(strs):
542 if s is not None:
543 big_i = mops.IntWiden(i)
544 d[big_i] = s
545 if mops.Greater(big_i, max_index):
546 max_index = big_i
547
548 return value.SparseArray(d, max_index)
549
550
551class SparseOp(vm._Callable):
552 """
553 All ops on value.SparseArray, for testing performance
554 """
555
556 def __init__(self):
557 # type: () -> None
558 pass
559
560 def Call(self, rd):
561 # type: (typed_args.Reader) -> value_t
562
563 sp = rd.PosSparseArray()
564 d = sp.d
565 #i = mops.BigTruncate(rd.PosInt())
566 op_name = rd.PosStr()
567
568 no_str = None # type: str
569
570 if op_name == 'len': # ${#a[@]}
571 rd.Done()
572 return num.ToBig(len(d))
573
574 elif op_name == 'get': # ${a[42]}
575 index = rd.PosInt()
576 rd.Done()
577
578 s = d.get(index)
579 if s is None:
580 return value.Null
581 else:
582 return value.Str(s)
583
584 elif op_name == 'set': # a[42]=foo
585 index = rd.PosInt()
586 s = rd.PosStr()
587 rd.Done()
588
589 d[index] = s
590
591 if mops.Greater(index, sp.max_index):
592 sp.max_index = index
593
594 return value.Int(mops.ZERO)
595
596 elif op_name == 'unset': # unset 'a[1]'
597 index = rd.PosInt()
598 rd.Done()
599
600 mylib.dict_erase(d, index)
601
602 max_index = mops.MINUS_ONE # Note: this works if d is not empty
603 for i1 in d:
604 if mops.Greater(i1, max_index): # i1 > max_index
605 max_index = i1
606 sp.max_index = max_index
607
608 return value.Int(mops.ZERO)
609
610 elif op_name == 'subst': # "${a[@]}"
611 # Algorithm to expand a Dict[BigInt, Str]
612 #
613 # 1. Copy the integer keys into a new List
614 # 2. Sort them in numeric order
615 # 3. Create a List[str] that's the same size as the keys
616 # 4. Loop through sorted keys, look up value, and populate list
617 #
618 # There is another possible algorithm:
619 #
620 # 1. Copy the VALUES into a new list
621 # 2. Somehow sort them by the CORRESPONDING key, which depends on
622 # Slab<> POSITION. I think this does not fit within the
623 # std::sort() model. I think we would have to write a little custom
624 # sort algorithm.
625
626 keys = d.keys()
627 mylib.BigIntSort(keys)
628 # Pre-allocate
629 items = [no_str] * len(d) # type: List[str]
630 j = 0
631 for i in keys:
632 s = d.get(i)
633 assert s is not None
634 items[j] = s
635 j += 1
636 return value.BashArray(items)
637
638 elif op_name == 'keys': # "${!a[@]}"
639 keys = d.keys()
640 mylib.BigIntSort(keys)
641 items = [mops.ToStr(k) for k in keys]
642
643 # TODO: return SparseArray
644 return value.BashArray(items)
645
646 elif op_name == 'slice': # "${a[@]:0:5}"
647 start = rd.PosInt()
648 end = rd.PosInt()
649 rd.Done()
650
651 n = mops.BigTruncate(mops.Sub(end, start))
652 #log('start %d - end %d', start.i, end.i)
653
654 # Pre-allocate
655 items2 = [no_str] * n # type: List[str]
656
657 # Iterate from start to end. Note that this algorithm is
658 # theoretically slower than bash in the case where the array is
659 # sparse (in the part selected by the slice)
660 #
661 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
662 # elements, OSH will iterate through 999 integers and do 999 dict
663 # lookups, while bash will follow 3 pointers.
664 #
665 # However, in practice, I think iterating through integers is
666 # cheap.
667
668 j = 0
669 i = start
670 while mops.Greater(end, i): # i < end
671 s = d.get(i)
672 #log('s %s', s)
673 if s is not None:
674 items2[j] = s
675 j += 1
676
677 i = mops.Add(i, mops.ONE) # i += 1
678
679 # TODO: return SparseArray
680 return value.BashArray(items2)
681
682 elif op_name == 'append': # a+=(x y)
683 strs = rd.PosBashArray()
684
685 # TODO: We can maintain the max index in the value.SparseArray(),
686 # so that it's O(1) to append rather than O(n)
687 # - Update on 'set' is O(1)
688 # - Update on 'unset' is potentially O(n)
689
690 if 0:
691 max_index = mops.MINUS_ONE # Note: this works for empty arrays
692 for i1 in d:
693 if mops.Greater(i1, max_index): # i1 > max_index
694 max_index = i1
695 else:
696 max_index = sp.max_index
697
698 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
699 for s in strs:
700 d[i2] = s
701 i2 = mops.Add(i2, mops.ONE) # i2 += 1
702
703 # sp.max_index += len(strs)
704 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
705 return value.Int(mops.ZERO)
706
707 else:
708 print('Invalid SparseArray operation %r' % op_name)
709 return value.Int(mops.ZERO)