# Backend class for the MoarVM. use MASTOps; my sub literal_subst(str $source, str $pattern, $replacement) { my $where := 0; my $result := $source; while (my $found := nqp::index($result, $pattern, $where)) != -1 { $where := $found + nqp::chars($replacement); $result := nqp::replace($result, $found, nqp::chars($pattern), $replacement); }; $result; } class HLL::Backend::MoarVM { our $StringHeap; our $Callsites; our %moar_config := nqp::backendconfig(); my sub read_ui32($fh, $buf?) { unless $buf { $buf := nqp::create($NQPBuf) } nqp::readfh($fh, $buf, 4); nqp::readuint($buf, 0, nqp::const::BINARY_SIZE_32_BIT); } my sub read_i16($fh, $buf?) { unless $buf { $buf := nqp::create($NQPBuf) } nqp::readfh($fh, $buf, 2); nqp::readint($buf, 0, nqp::const::BINARY_SIZE_16_BIT); } my sub read_confprog($path, $immediately_install = 0) { my $fh := nqp::open($path, "r"); my int $pos := 0; my str $comparison := "MOARVMCONFPROGVER001"; my $buf := nqp::readfh($fh, nqp::create($NQPBuf), nqp::chars($comparison)); while $pos < nqp::chars($comparison) { if nqp::atpos_i($buf, $pos) != nqp::ordbaseat($comparison, $pos) { nqp::die("Malformed confprog file, expected magic cookie at the start"); } $pos++; } my $decoder := NQPDecoder.new("utf8"); my @stringheap := nqp::list_s(); my int $stringcount := read_ui32($fh); $pos := 0; while $pos < $stringcount { my int $strlen := read_ui32($fh); if $strlen == 0 { nqp::push_s(@stringheap, ""); } else { my $strbuf := nqp::readfh($fh, nqp::create($NQPBuf), $strlen); $decoder.add-bytes($strbuf); nqp::push_s(@stringheap, my $str := $decoder.consume-all-chars()); unless $decoder.is-empty() { nqp::die("left-over bytes after decoding { nqp::elems(@stringheap) }st string"); } } $pos++; } my int $entrypointcount := read_ui32($fh); my @entrypoints := nqp::list_i(); $pos := 0; while $pos < $entrypointcount { nqp::push_i(@entrypoints, read_i16($fh)); $pos++; } my int $bytecodesize := read_ui32($fh); $buf := nqp::readfh($fh, $buf, $bytecodesize); if $immediately_install { #?if stage2 nqp::installconfprog($buf, @stringheap, @entrypoints); #?endif } else { return nqp::hash( "bytecode", $buf, "strings", @stringheap, "entrypoints", @entrypoints ); } } method config() { %moar_config } method force_gc() { nqp::force_gc(); } method name() { %moar_config } method nqpevent($spec?) { # Doesn't do anything just yet } method confprog($confprog-filename, *%adverbs) { read_confprog($confprog-filename, 1); } method profiler_snapshot(:$kind, :$filename) { if $kind eq "heap" { nqp::mvmstartprofile(nqp::hash("kind", "heap", "path", $filename)); return nqp::mvmendprofile(); } else { nqp::die("MoarVM's profiler_snapshot only supports kind => 'heap', not $kind"); } } my $prof_start_sub; my $prof_end_sub; method ensure_prof_routines() { unless $prof_start_sub { my %*COMPILING; self.start(''); $prof_start_sub := self.compunit_mainline(self.mbc(self.mast(QAST::CompUnit.new( QAST::Block.new( QAST::Op.new( :op('mvmstartprofile'), QAST::Var.new( :name('config'), :scope('local'), :decl('param') ) ) ))))); self.start(''); $prof_end_sub := self.compunit_mainline(self.mbc(self.mast(QAST::CompUnit.new( QAST::Block.new( QAST::Op.new( :op('mvmendprofile') ) ))))); } } method run_profiled($what, $filename, $kind) { unless $kind { if $filename ~~ / \. [ 'html' | 'json' | 'sql' ] $ / { $kind := 'instrumented'; } elsif $filename ~~ / '.mvmheap' $ / { $kind := 'heap'; } else { $kind := 'instrumented'; } } self.ensure_prof_routines(); my $conf-hash; if $kind eq "heap" { unless $filename { $filename := 'heap-snapshot-' ~ nqp::time() ~ '.mvmheap'; } $conf-hash := nqp::hash('kind', $kind, 'path', $filename); } else { unless $filename { $filename := 'profile-' ~ nqp::time() ~ '.html'; } $conf-hash := nqp::hash('kind', $kind); } my @END := nqp::gethllsym('Raku', '@END_PHASERS'); @END.push(-> { self.dump_profile_data($prof_end_sub(), $kind, $filename) }) if nqp::defined(@END); $prof_start_sub($conf-hash); my $res := $what(); unless nqp::defined(@END) { my $data := $prof_end_sub(); self.dump_profile_data($data, $kind, $filename); } $res; } method dump_profile_data($data, $kind, $filename) { if $kind eq 'instrumented' { self.dump_instrumented_profile_data($data, $filename); } elsif $kind eq 'heap' { self.dump_heap_profile_data($data, $filename); } else { nqp::die("Don't know how to dump data for $kind profile"); } } method dump_instrumented_profile_data($data, $filename) { my @pieces := nqp::list_s(); unless nqp::defined($filename) { $filename := 'profile-' ~ nqp::time() ~ '.html'; } note("Writing profiler output to $filename"); my $profile_fh; my $want_json := nqp::eqat($filename, '.json', -5); my $want_sql := nqp::eqat($filename, '.sql', -4); my $escaped_backslash; my $escaped_dquote; my $escaped_squote; if $want_json { # Single quotes don't require escaping here $escaped_backslash := q{\\\\}; $escaped_dquote := q{\\"}; } else { # Here we're creating a double-quoted JSON string destined for the # inside of a single-quoted JS string. Ouch. $escaped_backslash := q{\\\\\\\\}; $escaped_dquote := q{\\\\"}; $escaped_squote := q{\\'}; } my int $new-id-counter := -1; my $id_remap := nqp::hash(); my $id_to_thing := nqp::hash(); my %type-info := nqp::hash(); my int $node-id-counter := -1; sub get_remapped_type_id($id) { my str $newkey; if nqp::existskey($id_remap, $id) { $newkey := $id_remap{$id}; } else { $newkey := ~(++$new-id-counter); $id_remap{$id} := $newkey; } unless nqp::existskey($id_to_thing, $newkey) { my $typename; my $scdesc; try { my $type := %type-info{$id}[1]; $typename := $type.HOW.name($type); } unless $typename { $typename := ""; } try { my $type := %type-info{$id}[1]; my $sc := nqp::getobjsc($type); if $sc { $scdesc := nqp::scgetdesc($sc); } } unless $scdesc { $scdesc := ""; } %type-info{$id}[1] := $typename; %type-info{$id}[1] := $scdesc; $id_to_thing{$newkey} := $typename; unless nqp::existskey(%type-info, $newkey) { nqp::bindkey(%type-info, $newkey, nqp::list()); } %type-info{$newkey}[1] := %type-info{$id}[1]; } $newkey; } sub post_process_call_graph_node($node) { my $this-node-id := ++$node-id-counter; try { if nqp::existskey($id_remap, $node) { $node := $id_remap{$node}; } else { my str $newkey := ~(++$new-id-counter); $id_remap{$node} := $newkey; $node := $newkey; } if nqp::existskey($node, "allocations") { for $node -> %alloc_info { %alloc_info := get_remapped_type_id(%alloc_info); nqp::deletekey(%alloc_info, "type"); } } unless nqp::existskey($id_to_thing, $node) { my $shared_data := nqp::hash( "file", $node, "line", $node, "name", $node, ); $id_to_thing{$node} := $shared_data; } nqp::deletekey($node, "file"); nqp::deletekey($node, "line"); nqp::deletekey($node, "name"); if nqp::existskey($node, "callees") { for $node { post_process_call_graph_node($_); } } CATCH { note("profiler caught an error during post_process_call_graph_node:"); note(nqp::getmessage($!)); } } $node := $node-id-counter; } sub post_process_thread_data($thread) { unless nqp::existskey($thread, 'gcs') { return } for $thread -> $gc { if nqp::existskey($gc, 'deallocs') { for $gc -> $dealloc { $dealloc := get_remapped_type_id($dealloc); } } } } sub to_json($obj) { if nqp::islist($obj) { nqp::push_s(@pieces, '['); my int $first := 1; for $obj { if $first { $first := 0; } else { nqp::push_s(@pieces, ','); } to_json($_); } nqp::push_s(@pieces, ']'); } elsif nqp::ishash($obj) { nqp::push_s(@pieces, '{'); my int $first := 1; for sorted_keys($obj) { if $first { $first := 0; } else { nqp::push_s(@pieces, ','); } nqp::push_s(@pieces, '"'); nqp::push_s(@pieces, $_); nqp::push_s(@pieces, '":'); to_json($obj{$_}); } nqp::push_s(@pieces, '}'); } elsif nqp::isstr($obj) { if nqp::index($obj, '\\') { $obj := literal_subst($obj, '\\', $escaped_backslash); } if nqp::index($obj, '"') { $obj := literal_subst($obj, '"', $escaped_dquote); } if nqp::defined($escaped_squote) && nqp::index($obj, "'") { $obj := literal_subst($obj, "'", $escaped_squote); } nqp::push_s(@pieces, '"'); nqp::push_s(@pieces, $obj); nqp::push_s(@pieces, '"'); } elsif nqp::isint($obj) || nqp::isnum($obj) { nqp::push_s(@pieces, ~$obj); } elsif nqp::can($obj, 'Str') { to_json(nqp::unbox_s($obj.Str)); } else { nqp::die("Don't know how to dump a " ~ $obj.HOW.name($obj)); } if nqp::elems(@pieces) > 4096 { $profile_fh.print(nqp::join('', @pieces)); nqp::setelems(@pieces, 0); } } sub to_sql_json($obj, $pieces?) { my $will-return := !nqp::isconcrete($pieces); unless nqp::isconcrete($obj) { if $will-return { return "null"; } nqp::push_s($pieces, "null"); return; } unless nqp::isconcrete($pieces) { $pieces := nqp::list_s; } if nqp::islist($obj) { nqp::push_s($pieces, 'json_array('); my int $first := 1; for $obj { if $first { $first := 0; } else { nqp::push_s($pieces, ','); } to_sql_json($_, $pieces); } nqp::push_s($pieces, ')'); } elsif nqp::ishash($obj) { nqp::push_s($pieces, 'json_object('); my int $first := 1; for sorted_keys($obj) { if $first { $first := 0; } else { nqp::push_s($pieces, ','); } nqp::push_s($pieces, "'"); nqp::push_s($pieces, $_); nqp::push_s($pieces, "', "); to_sql_json($obj{$_}, $pieces); } nqp::push_s($pieces, ')'); } elsif nqp::isstr($obj) { if nqp::index($obj, '\\') { $obj := literal_subst($obj, '\\', '\\\\'); } if nqp::index($obj, '"') { $obj := literal_subst($obj, '"', '\\"'); } if nqp::index($obj, "'") { $obj := literal_subst($obj, "'", "\\'"); } nqp::push_s($pieces, "'"); nqp::push_s($pieces, $obj); nqp::push_s($pieces, "'"); } elsif nqp::isint($obj) || nqp::isnum($obj) { nqp::push_s($pieces, ~$obj); } elsif nqp::can($obj, 'Str') { to_sql_json(nqp::unbox_s($obj.Str), $pieces); } else { nqp::push_s($pieces, 'null'); } if $will-return { return nqp::join("", $pieces); } } sub to_sql($obj) { my int $node_id := 0; #my %profile := nqp::hash(); my $mapping := nqp::shift($obj); my $pieces := nqp::list_s(); my $empty-array := nqp::list_s(); nqp::push_s($pieces, "INSERT INTO routines VALUES ('"); my $is-first := 1; for $mapping -> $k { my $v := $mapping{$k}; if nqp::ishash($v) { if !$is-first { nqp::push_s($pieces, ", ('"); } else { $is-first := 0 } nqp::push_s($pieces, nqp::join("','", nqp::list( nqp::iterkey_s($k), literal_subst(~$v, "'", "''"), ~$v, ~$v)) ~ "')"); } if nqp::elems($pieces) > 500 { $profile_fh.print(nqp::join("", $pieces)); nqp::splice($pieces, $empty-array, 0, nqp::elems($pieces)); } } nqp::push_s($pieces, ";\n"); $is-first := 1; nqp::push_s($pieces, "INSERT INTO types VALUES ('"); for $mapping -> $k { my $v := $mapping{$k}; if !nqp::ishash($v) { if !$is-first { nqp::push_s($pieces, ", ('"); } else { $is-first := 0 } my $type-info := %type-info{nqp::iterkey_s($k)}; nqp::push_s($pieces, nqp::join("','", nqp::list_s( $k, literal_subst(~$v, "'", "''"), )) ~ "'," ~ to_sql_json($type-info[1]) ~ "," ~ "json_object()" ~ ")"); } if nqp::elems($pieces) > 500 { $profile_fh.print(nqp::join("", $pieces)); nqp::splice($pieces, $empty-array, 0, nqp::elems($pieces)); } } nqp::push_s($pieces, ";\n"); for $obj -> $thread { my $thisprof := nqp::list; $thisprof[4] := "NULL"; $thisprof[5] := ~$thread; for $thread -> $k { my $v := $thread{$k}; if $k eq 'total_time' { $thisprof[0] := ~$v; } elsif $k eq 'spesh_time' { $thisprof[1] := ~$v; } elsif $k eq 'thread' { $thisprof[2] := ~$v; } elsif $k eq 'gcs' { my str $thread_id := $thread; if nqp::elems($v) > 0 { nqp::push_s($pieces, "INSERT INTO gcs VALUES ("); } my $any-deallocs := 0; my $is-first := 1; for $v -> $gc { if !$is-first { nqp::push_s($pieces, ", ("); } else { $is-first := 0 } my @g := nqp::list_s(); for