OILS / vendor / souffle / profile / EventProcessor.h View on Github | oilshell.org

612 lines, 394 significant
1/*
2 * Souffle - A Datalog Compiler
3 * Copyright (c) 2018, The Souffle Developers. All rights reserved.
4 * Licensed under the Universal Permissive License v 1.0 as shown at:
5 * - https://opensource.org/licenses/UPL
6 * - <souffle root>/licenses/SOUFFLE-UPL.txt
7 */
8
9/************************************************************************
10 *
11 * @file EventProcessor.h
12 *
13 * Declares classes for event processor that parse profile events and
14 * populate the profile database
15 *
16 ***********************************************************************/
17
18#pragma once
19
20#include "souffle/profile/ProfileDatabase.h"
21#include "souffle/utility/MiscUtil.h"
22#include "souffle/utility/StreamUtil.h"
23#include <cassert>
24#include <chrono>
25#include <cstdarg>
26#include <cstdint>
27#include <cstdlib>
28#include <iostream>
29#include <map>
30#include <string>
31#include <vector>
32
33namespace souffle {
34namespace profile {
35/**
36 * Abstract Class for EventProcessor
37 */
38class EventProcessor {
39public:
40 virtual ~EventProcessor() = default;
41
42 /** abstract interface for processing an profile event */
43 virtual void process(ProfileDatabase&, const std::vector<std::string>& signature, va_list&) {
44 fatal("Unknown profiling processing event: %s", join(signature, " "));
45 }
46};
47
48/**
49 * Event Processor Singleton
50 *
51 * Singleton that is the connection point for events
52 */
53class EventProcessorSingleton {
54public:
55 /** get instance */
56 static EventProcessorSingleton& instance() {
57 static EventProcessorSingleton singleton;
58 return singleton;
59 }
60
61 /** register an event processor with its keyword */
62 void registerEventProcessor(const std::string& keyword, EventProcessor* processor) {
63 registry[keyword] = processor;
64 }
65
66 /** process a profile event */
67 void process(ProfileDatabase& db, const char* txt, ...) {
68 va_list args;
69 va_start(args, txt);
70
71 // escape signature
72 std::string escapedText = escape(txt);
73 // obtain event signature by splitting event text
74 std::vector<std::string> eventSignature = splitSignature(escapedText);
75
76 // invoke the event processor of the event
77 const std::string& keyword = eventSignature[0];
78 assert(eventSignature.size() > 0 && "no keyword in event description");
79 assert(registry.find(keyword) != registry.end() && "EventProcessor not found!");
80 registry[keyword]->process(db, eventSignature, args);
81
82 // terminate access to variadic arguments
83 va_end(args);
84 }
85
86private:
87 /** keyword / event processor mapping */
88 std::map<std::string, EventProcessor*> registry;
89
90 EventProcessorSingleton() = default;
91
92 /**
93 * Escape escape characters.
94 *
95 * Remove all escapes, then escape double quotes.
96 */
97 std::string escape(const std::string& text) {
98 std::string str(text);
99 std::size_t start_pos = 0;
100 // replace backslashes with double backslash
101 while ((start_pos = str.find('\\', start_pos)) != std::string::npos) {
102 if (start_pos == str.size()) {
103 break;
104 }
105 ++start_pos;
106 if (str[start_pos] != 't' && str[start_pos] != '"' && str[start_pos] != '\\' &&
107 str[start_pos] != 'n' && str[start_pos] != ';') {
108 str.replace(start_pos - 1, 1, "\\\\");
109 }
110 ++start_pos;
111 }
112 return str;
113 }
114
115 /** split string */
116 static std::vector<std::string> split(std::string str, std::string split_str) {
117 // repeat value when splitting so "a b" -> ["a","b"] not ["a","","","","b"]
118 bool repeat = (split_str == " ");
119
120 std::vector<std::string> elems;
121
122 std::string temp;
123 std::string hold;
124 for (std::size_t i = 0; i < str.size(); i++) {
125 if (repeat) {
126 if (str.at(i) == split_str.at(0)) {
127 while (str.at(++i) == split_str.at(0)) {
128 ; // set i to be at the end of the search string
129 }
130 elems.push_back(temp);
131 temp = "";
132 }
133 temp += str.at(i);
134 } else {
135 temp += str.at(i);
136 hold += str.at(i);
137 for (std::size_t j = 0; j < hold.size(); j++) {
138 if (hold[j] != split_str[j]) {
139 hold = "";
140 }
141 }
142 if (hold.size() == split_str.size()) {
143 elems.push_back(temp.substr(0, temp.size() - hold.size()));
144 hold = "";
145 temp = "";
146 }
147 }
148 }
149 if (!temp.empty()) {
150 elems.push_back(temp);
151 }
152
153 return elems;
154 }
155
156 /** split string separated by semi-colon */
157 static std::vector<std::string> splitSignature(std::string str) {
158 for (std::size_t i = 0; i < str.size(); i++) {
159 if (i > 0 && str[i] == ';' && str[i - 1] == '\\') {
160 // I'm assuming this isn't a thing that will be naturally found in souffle profiler files
161 str[i - 1] = '\b';
162 str.erase(i--, 1);
163 }
164 }
165 std::vector<std::string> result = split(str, ";");
166 for (auto& i : result) {
167 for (char& j : i) {
168 if (j == '\b') {
169 j = ';';
170 }
171 }
172 }
173 return result;
174 }
175};
176
177/**
178 * Non-Recursive Rule Timing Profile Event Processor
179 */
180const class NonRecursiveRuleTimingProcessor : public EventProcessor {
181public:
182 NonRecursiveRuleTimingProcessor() {
183 EventProcessorSingleton::instance().registerEventProcessor("@t-nonrecursive-rule", this);
184 }
185 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
186 const std::string& relation = signature[1];
187 const std::string& srcLocator = signature[2];
188 const std::string& rule = signature[3];
189 microseconds start = va_arg(args, microseconds);
190 microseconds end = va_arg(args, microseconds);
191 std::size_t startMaxRSS = va_arg(args, std::size_t);
192 std::size_t endMaxRSS = va_arg(args, std::size_t);
193 std::size_t size = va_arg(args, std::size_t);
194 db.addSizeEntry(
195 {"program", "relation", relation, "non-recursive-rule", rule, "maxRSS", "pre"}, startMaxRSS);
196 db.addSizeEntry(
197 {"program", "relation", relation, "non-recursive-rule", rule, "maxRSS", "post"}, endMaxRSS);
198 db.addTextEntry(
199 {"program", "relation", relation, "non-recursive-rule", rule, "source-locator"}, srcLocator);
200 db.addDurationEntry(
201 {"program", "relation", relation, "non-recursive-rule", rule, "runtime"}, start, end);
202 db.addSizeEntry({"program", "relation", relation, "non-recursive-rule", rule, "num-tuples"}, size);
203 }
204} nonRecursiveRuleTimingProcessor;
205
206/**
207 * Non-Recursive Rule Number Profile Event Processor
208 */
209const class NonRecursiveRuleNumberProcessor : public EventProcessor {
210public:
211 NonRecursiveRuleNumberProcessor() {
212 EventProcessorSingleton::instance().registerEventProcessor("@n-nonrecursive-rule", this);
213 }
214 /** process event input */
215 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
216 const std::string& relation = signature[1];
217 const std::string& srcLocator = signature[2];
218 const std::string& rule = signature[3];
219 std::size_t num = va_arg(args, std::size_t);
220 db.addTextEntry(
221 {"program", "relation", relation, "non-recursive-rule", rule, "source-locator"}, srcLocator);
222 db.addSizeEntry({"program", "relation", relation, "non-recursive-rule", rule, "num-tuples"}, num);
223 }
224} nonRecursiveRuleNumberProcessor;
225
226/**
227 * Non-Recursive Estimate Join Size Profile Event Processor
228 */
229const class NonRecursiveEstimateJoinSizeProcessor : public EventProcessor {
230public:
231 NonRecursiveEstimateJoinSizeProcessor() {
232 EventProcessorSingleton::instance().registerEventProcessor("@non-recursive-estimate-join-size", this);
233 }
234 /** process event input */
235 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
236 const std::string& relation = signature[1];
237 const std::string& attributes = signature[2];
238 const std::string& constants = signature[3];
239 std::string joinSize = std::to_string(va_arg(args, double));
240 db.addTextEntry({"program", "statistics", "relation", relation, "attributes", attributes, "constants",
241 constants},
242 joinSize);
243 }
244} nonRecursiveEstimateJoinSizeProcessor;
245
246/**
247 * Recursive Estimate Join Size Profile Event Processor
248 */
249const class RecursiveEstimateJoinSizeProcessor : public EventProcessor {
250public:
251 RecursiveEstimateJoinSizeProcessor() {
252 EventProcessorSingleton::instance().registerEventProcessor("@recursive-estimate-join-size", this);
253 }
254 /** process event input */
255 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
256 const std::string& relation = signature[1];
257 const std::string& attributes = signature[2];
258 const std::string& constants = signature[3];
259 std::string joinSize = std::to_string(va_arg(args, double));
260 std::string iteration = std::to_string(va_arg(args, std::size_t));
261 db.addTextEntry({"program", "statistics", "relation", relation, "iteration", iteration, "attributes",
262 attributes, "constants", constants},
263 joinSize);
264 }
265} recursiveEstimateJoinSizeProcessor;
266
267/**
268 * Recursive Rule Timing Profile Event Processor
269 */
270const class RecursiveRuleTimingProcessor : public EventProcessor {
271public:
272 RecursiveRuleTimingProcessor() {
273 EventProcessorSingleton::instance().registerEventProcessor("@t-recursive-rule", this);
274 }
275 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
276 const std::string& relation = signature[1];
277 const std::string& version = signature[2];
278 const std::string& srcLocator = signature[3];
279 const std::string& rule = signature[4];
280 microseconds start = va_arg(args, microseconds);
281 microseconds end = va_arg(args, microseconds);
282 std::size_t startMaxRSS = va_arg(args, std::size_t);
283 std::size_t endMaxRSS = va_arg(args, std::size_t);
284 std::size_t size = va_arg(args, std::size_t);
285 std::string iteration = std::to_string(va_arg(args, std::size_t));
286 db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,
287 version, "maxRSS", "pre"},
288 startMaxRSS);
289 db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,
290 version, "maxRSS", "post"},
291 endMaxRSS);
292 db.addTextEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,
293 version, "source-locator"},
294 srcLocator);
295 db.addDurationEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,
296 version, "runtime"},
297 start, end);
298 db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,
299 version, "num-tuples"},
300 size);
301 }
302} recursiveRuleTimingProcessor;
303
304/**
305 * Recursive Rule Number Profile Event Processor
306 */
307const class RecursiveRuleNumberProcessor : public EventProcessor {
308public:
309 RecursiveRuleNumberProcessor() {
310 EventProcessorSingleton::instance().registerEventProcessor("@n-recursive-rule", this);
311 }
312 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
313 const std::string& relation = signature[1];
314 const std::string& version = signature[2];
315 const std::string& srcLocator = signature[3];
316 const std::string& rule = signature[4];
317 std::size_t number = va_arg(args, std::size_t);
318 std::string iteration = std::to_string(va_arg(args, std::size_t));
319 db.addTextEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,
320 version, "source-locator"},
321 srcLocator);
322 db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,
323 version, "num-tuples"},
324 number);
325 }
326} recursiveRuleNumberProcessor;
327
328/**
329 * Non-Recursive Relation Number Profile Event Processor
330 */
331const class NonRecursiveRelationTimingProcessor : public EventProcessor {
332public:
333 NonRecursiveRelationTimingProcessor() {
334 EventProcessorSingleton::instance().registerEventProcessor("@t-nonrecursive-relation", this);
335 }
336 /** process event input */
337 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
338 const std::string& relation = signature[1];
339 const std::string& srcLocator = signature[2];
340 microseconds start = va_arg(args, microseconds);
341 microseconds end = va_arg(args, microseconds);
342 std::size_t startMaxRSS = va_arg(args, std::size_t);
343 std::size_t endMaxRSS = va_arg(args, std::size_t);
344 std::size_t size = va_arg(args, std::size_t);
345 db.addSizeEntry({"program", "relation", relation, "maxRSS", "pre"}, startMaxRSS);
346 db.addSizeEntry({"program", "relation", relation, "maxRSS", "post"}, endMaxRSS);
347 db.addSizeEntry({"program", "relation", relation, "num-tuples"}, size);
348 db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);
349 db.addDurationEntry({"program", "relation", relation, "runtime"}, start, end);
350 }
351} nonRecursiveRelationTimingProcessor;
352
353/**
354 * Non-Recursive Relation Number Profile Event Processor
355 */
356const class NonRecursiveRelationNumberProcessor : public EventProcessor {
357public:
358 NonRecursiveRelationNumberProcessor() {
359 EventProcessorSingleton::instance().registerEventProcessor("@n-nonrecursive-relation", this);
360 }
361 /** process event input */
362 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
363 const std::string& relation = signature[1];
364 const std::string& srcLocator = signature[2];
365 std::size_t num = va_arg(args, std::size_t);
366 db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);
367 db.addSizeEntry({"program", "relation", relation, "num-tuples"}, num);
368 }
369} nonRecursiveRelationNumberProcessor;
370
371/**
372 * Recursive Relation Timing Profile Event Processor
373 */
374const class RecursiveRelationTimingProcessor : public EventProcessor {
375public:
376 RecursiveRelationTimingProcessor() {
377 EventProcessorSingleton::instance().registerEventProcessor("@t-recursive-relation", this);
378 }
379 /** process event input */
380 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
381 const std::string& relation = signature[1];
382 const std::string& srcLocator = signature[2];
383 microseconds start = va_arg(args, microseconds);
384 microseconds end = va_arg(args, microseconds);
385 std::size_t startMaxRSS = va_arg(args, std::size_t);
386 std::size_t endMaxRSS = va_arg(args, std::size_t);
387 std::size_t size = va_arg(args, std::size_t);
388 std::string iteration = std::to_string(va_arg(args, std::size_t));
389 db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);
390 db.addDurationEntry({"program", "relation", relation, "iteration", iteration, "runtime"}, start, end);
391 db.addSizeEntry(
392 {"program", "relation", relation, "iteration", iteration, "maxRSS", "pre"}, startMaxRSS);
393 db.addSizeEntry(
394 {"program", "relation", relation, "iteration", iteration, "maxRSS", "post"}, endMaxRSS);
395 db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "num-tuples"}, size);
396 }
397} recursiveRelationTimingProcessor;
398
399/**
400 * Recursive Relation Timing Profile Event Processor
401 */
402const class RecursiveRelationNumberProcessor : public EventProcessor {
403public:
404 RecursiveRelationNumberProcessor() {
405 EventProcessorSingleton::instance().registerEventProcessor("@n-recursive-relation", this);
406 }
407 /** process event input */
408 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
409 const std::string& relation = signature[1];
410 const std::string& srcLocator = signature[2];
411 std::size_t number = va_arg(args, std::size_t);
412 std::string iteration = std::to_string(va_arg(args, std::size_t));
413 db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);
414 db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "num-tuples"}, number);
415 }
416} recursiveRelationNumberProcessor;
417
418/**
419 * Recursive Relation Copy Timing Profile Event Processor
420 */
421const class RecursiveRelationCopyTimingProcessor : public EventProcessor {
422public:
423 RecursiveRelationCopyTimingProcessor() {
424 EventProcessorSingleton::instance().registerEventProcessor("@c-recursive-relation", this);
425 }
426 /** process event input */
427 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
428 const std::string& relation = signature[1];
429 const std::string& srcLocator = signature[2];
430 microseconds start = va_arg(args, microseconds);
431 microseconds end = va_arg(args, microseconds);
432 std::size_t startMaxRSS = va_arg(args, std::size_t);
433 std::size_t endMaxRSS = va_arg(args, std::size_t);
434 va_arg(args, std::size_t);
435 std::string iteration = std::to_string(va_arg(args, std::size_t));
436 db.addSizeEntry(
437 {"program", "relation", relation, "iteration", iteration, "maxRSS", "pre"}, startMaxRSS);
438 db.addSizeEntry(
439 {"program", "relation", relation, "iteration", iteration, "maxRSS", "post"}, endMaxRSS);
440 db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);
441 db.addDurationEntry(
442 {"program", "relation", relation, "iteration", iteration, "copytime"}, start, end);
443 }
444} recursiveRelationCopyTimingProcessor;
445
446/**
447 * Recursive Relation Copy Timing Profile Event Processor
448 */
449const class RelationIOTimingProcessor : public EventProcessor {
450public:
451 RelationIOTimingProcessor() {
452 EventProcessorSingleton::instance().registerEventProcessor("@t-relation-savetime", this);
453 EventProcessorSingleton::instance().registerEventProcessor("@t-relation-loadtime", this);
454 }
455 /** process event input */
456 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
457 const std::string& relation = signature[1];
458 const std::string& srcLocator = signature[2];
459 const std::string ioType = signature[3];
460 microseconds start = va_arg(args, microseconds);
461 microseconds end = va_arg(args, microseconds);
462 db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);
463 db.addDurationEntry({"program", "relation", relation, ioType}, start, end);
464 }
465} relationIOTimingProcessor;
466
467/**
468 * Program Run Event Processor
469 */
470const class ProgramTimepointProcessor : public EventProcessor {
471public:
472 ProgramTimepointProcessor() {
473 EventProcessorSingleton::instance().registerEventProcessor("@time", this);
474 }
475 /** process event input */
476 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
477 microseconds time = va_arg(args, microseconds);
478 auto path = signature;
479 path[0] = "program";
480 db.addTimeEntry(path, time);
481 }
482} programTimepointProcessor;
483
484/**
485 * Program Run Event Processor
486 */
487const class ProgramRuntimeProcessor : public EventProcessor {
488public:
489 ProgramRuntimeProcessor() {
490 EventProcessorSingleton::instance().registerEventProcessor("@runtime", this);
491 }
492 /** process event input */
493 void process(
494 ProfileDatabase& db, const std::vector<std::string>& /* signature */, va_list& args) override {
495 microseconds start = va_arg(args, microseconds);
496 microseconds end = va_arg(args, microseconds);
497 db.addDurationEntry({"program", "runtime"}, start, end);
498 }
499} programRuntimeProcessor;
500
501/**
502 * Program Resource Utilisation Event Processor
503 */
504const class ProgramResourceUtilisationProcessor : public EventProcessor {
505public:
506 ProgramResourceUtilisationProcessor() {
507 EventProcessorSingleton::instance().registerEventProcessor("@utilisation", this);
508 }
509 /** process event input */
510 void process(
511 ProfileDatabase& db, const std::vector<std::string>& /* signature */, va_list& args) override {
512 microseconds time = va_arg(args, microseconds);
513 uint64_t systemTime = va_arg(args, uint64_t);
514 uint64_t userTime = va_arg(args, uint64_t);
515 std::size_t maxRSS = va_arg(args, std::size_t);
516 std::string timeString = std::to_string(time.count());
517 db.addSizeEntry({"program", "usage", "timepoint", timeString, "systemtime"}, systemTime);
518 db.addSizeEntry({"program", "usage", "timepoint", timeString, "usertime"}, userTime);
519 db.addSizeEntry({"program", "usage", "timepoint", timeString, "maxRSS"}, maxRSS);
520 }
521} programResourceUtilisationProcessor;
522
523/**
524 * Frequency Atom Processor
525 */
526const class FrequencyAtomProcessor : public EventProcessor {
527public:
528 FrequencyAtomProcessor() {
529 EventProcessorSingleton::instance().registerEventProcessor("@frequency-atom", this);
530 }
531 /** process event input */
532 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
533 const std::string& relation = signature[1];
534 const std::string& version = signature[2];
535 const std::string& rule = signature[3];
536 const std::string& atom = signature[4];
537 const std::string& originalRule = signature[5];
538 std::size_t level = std::stoi(signature[6]);
539 std::size_t number = va_arg(args, std::size_t);
540 std::size_t iteration = va_arg(args, std::size_t);
541 // non-recursive rule
542 if (rule == originalRule) {
543 db.addSizeEntry({"program", "relation", relation, "non-recursive-rule", rule, "atom-frequency",
544 rule, atom, "level"},
545 level);
546 db.addSizeEntry({"program", "relation", relation, "non-recursive-rule", rule, "atom-frequency",
547 rule, atom, "num-tuples"},
548 number);
549 } else {
550 db.addSizeEntry(
551 {"program", "relation", relation, "iteration", std::to_string(iteration),
552 "recursive-rule", originalRule, version, "atom-frequency", rule, atom, "level"},
553 level);
554 db.addSizeEntry({"program", "relation", relation, "iteration", std::to_string(iteration),
555 "recursive-rule", originalRule, version, "atom-frequency", rule, atom,
556 "num-tuples"},
557 number);
558 }
559 }
560} frequencyAtomProcessor;
561
562/**
563 * Reads Processor
564 */
565const class RelationReadsProcessor : public EventProcessor {
566public:
567 RelationReadsProcessor() {
568 EventProcessorSingleton::instance().registerEventProcessor("@relation-reads", this);
569 }
570 /** process event input */
571 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
572 const std::string& relation = signature[1];
573 std::size_t reads = va_arg(args, std::size_t);
574 db.addSizeEntry({"program", "relation", relation, "reads"}, reads);
575 }
576
577} relationReadsProcessor;
578
579/**
580 * Config entry processor
581 */
582const class ConfigProcessor : public EventProcessor {
583public:
584 ConfigProcessor() {
585 EventProcessorSingleton::instance().registerEventProcessor("@config", this);
586 }
587 void process(
588 ProfileDatabase& db, const std::vector<std::string>& /* signature */, va_list& args) override {
589 const std::string key = va_arg(args, char*);
590 const std::string& value = va_arg(args, char*);
591 db.addTextEntry({"program", "configuration", key}, value);
592 }
593} configProcessor;
594
595/**
596 * Text entry processor
597 */
598const class TextProcessor : public EventProcessor {
599public:
600 TextProcessor() {
601 EventProcessorSingleton::instance().registerEventProcessor("@text", this);
602 }
603 void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {
604 const std::string text = va_arg(args, char*);
605 auto path = signature;
606 path.front() = "program";
607 db.addTextEntry(path, text);
608 }
609} textProcessor;
610
611} // namespace profile
612} // namespace souffle