From d99cbde1c90802ae77fe002196650d01f272cf3a Mon Sep 17 00:00:00 2001 From: Josh Pschorr Date: Mon, 25 Jul 2022 10:24:25 -0700 Subject: [PATCH 1/2] Add visualization of ASTs to CLI & REPL - Add actual arg parsing to CLI - Make REPL a sub-mode initiated by a CLI command - Add partial (lots of `todo!()`s) translation of AST into graphviz - Add FFI to graphviz to layout & render - Add `display` mode which prints image to console --- partiql-cli/Cargo.toml | 41 +- partiql-cli/src/args.rs | 38 ++ partiql-cli/src/error.rs | 82 +++ partiql-cli/src/{bin => }/ion.sublime-syntax | 0 partiql-cli/src/lib.rs | 7 + partiql-cli/src/main.rs | 43 ++ .../src/{bin => }/partiql.sublime-syntax | 0 .../src/{bin/partiql-cli.rs => repl.rs} | 113 +--- partiql-cli/src/visualize/ast_to_dot.rs | 589 ++++++++++++++++++ partiql-cli/src/visualize/mod.rs | 2 + partiql-cli/src/visualize/render.rs | 132 ++++ 11 files changed, 951 insertions(+), 96 deletions(-) create mode 100644 partiql-cli/src/args.rs create mode 100644 partiql-cli/src/error.rs rename partiql-cli/src/{bin => }/ion.sublime-syntax (100%) create mode 100644 partiql-cli/src/lib.rs create mode 100644 partiql-cli/src/main.rs rename partiql-cli/src/{bin => }/partiql.sublime-syntax (100%) rename partiql-cli/src/{bin/partiql-cli.rs => repl.rs} (65%) create mode 100644 partiql-cli/src/visualize/ast_to_dot.rs create mode 100644 partiql-cli/src/visualize/mod.rs create mode 100644 partiql-cli/src/visualize/render.rs diff --git a/partiql-cli/Cargo.toml b/partiql-cli/Cargo.toml index 24e85478..19ebbf24 100644 --- a/partiql-cli/Cargo.toml +++ b/partiql-cli/Cargo.toml @@ -41,14 +41,45 @@ supports-unicode = "1.0.2" supports-hyperlinks = "1.2.0" termbg = "0.4.1" shellexpand = "2.1.0" -partiql-parser = { path = "../partiql-parser" } -partiql-source-map = { path = "../partiql-source-map" } -partiql-ast = { path = "../partiql-ast" } thiserror = "1.0.31" miette = { version ="4.7.1", features = ["fancy"] } +clap = { version = "3.2.13", features = ["derive"] } + +# serde +serde = { version ="1.*", features = ["derive"], optional = true } +serde_json = { version ="1.*", optional = true } +### Dependencies for the `render` feature +viuer = { version ="0.6.1", features = ["sixel"], optional = true } +image = { version ="0.24.2", optional = true } +graphviz-sys = { version ="0.1.3", optional = true } +resvg = { version ="0.23.0", optional = true } +usvg = { version ="0.23.0", optional = true } +tiny-skia = { version ="0.6.6", optional = true } +strum = { version ="0.24.1", features = ["derive"], optional = true } +dot-writer = { version = "~0.1.2", optional = true } -tui = "0.18.0" -crossterm = "0.23.2" \ No newline at end of file + + +[features] +default = [] +serde = [ + "dep:serde", + "dep:serde_json", + "partiql-parser/serde", + "partiql-source-map/serde", + "partiql-ast/serde", +] +visualize = [ + "serde", + "dep:viuer", + "dep:image", + "dep:graphviz-sys", + "dep:resvg", + "dep:usvg", + "dep:tiny-skia", + "dep:strum", + "dep:dot-writer", +] diff --git a/partiql-cli/src/args.rs b/partiql-cli/src/args.rs new file mode 100644 index 00000000..331b5d08 --- /dev/null +++ b/partiql-cli/src/args.rs @@ -0,0 +1,38 @@ +use clap::{ArgEnum, Parser, Subcommand}; + +#[derive(Parser)] +#[clap(author, version, about, long_about = None)] +pub struct Args { + #[clap(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand)] +pub enum Commands { + #[cfg(feature = "visualize")] + /// Dump the AST for a query + Ast { + #[clap(short = 'T', long = "format", value_enum)] + format: Format, + + /// Query to parse + #[clap(value_parser)] + query: String, + }, + /// interactive REPL (Read Eval Print Loop) shell + Repl, +} + +#[derive(ArgEnum, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum Format { + /// JSON + Json, + /// Graphviz dot + Dot, + /// Graphviz svg output + Svg, + /// Graphviz svg rendered to png + Png, + /// Display rendered output + Display, +} diff --git a/partiql-cli/src/error.rs b/partiql-cli/src/error.rs new file mode 100644 index 00000000..69640e20 --- /dev/null +++ b/partiql-cli/src/error.rs @@ -0,0 +1,82 @@ +use miette::{Diagnostic, LabeledSpan, SourceCode}; +use partiql_parser::ParseError; +use partiql_source_map::location::{BytePosition, Location}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum CLIError { + #[error("PartiQL syntax error:")] + SyntaxError { + src: String, + msg: String, + loc: Location, + }, + // TODO add github issue link + #[error("Internal Compiler Error - please report this.")] + InternalCompilerError { src: String }, +} + +impl Diagnostic for CLIError { + fn source_code(&self) -> Option<&dyn SourceCode> { + match self { + CLIError::SyntaxError { src, .. } => Some(src), + CLIError::InternalCompilerError { src, .. } => Some(src), + } + } + + fn labels(&self) -> Option + '_>> { + match self { + CLIError::SyntaxError { msg, loc, .. } => { + Some(Box::new(std::iter::once(LabeledSpan::new( + Some(msg.to_string()), + loc.start.0 .0 as usize, + loc.end.0 .0 as usize - loc.start.0 .0 as usize, + )))) + } + CLIError::InternalCompilerError { .. } => None, + } + } +} + +impl CLIError { + pub fn from_parser_error(err: ParseError, source: &str) -> CLIError { + match err { + ParseError::SyntaxError(partiql_source_map::location::Located { inner, location }) => { + CLIError::SyntaxError { + src: source.to_string(), + msg: format!("Syntax error `{}`", inner), + loc: location, + } + } + ParseError::UnexpectedToken(partiql_source_map::location::Located { + inner, + location, + }) => CLIError::SyntaxError { + src: source.to_string(), + msg: format!("Unexpected token `{}`", inner.token), + loc: location, + }, + ParseError::LexicalError(partiql_source_map::location::Located { inner, location }) => { + CLIError::SyntaxError { + src: source.to_string(), + msg: format!("Lexical error `{}`", inner), + loc: location, + } + } + ParseError::Unknown(location) => CLIError::SyntaxError { + src: source.to_string(), + msg: "Unknown parser error".to_string(), + loc: Location { + start: location, + end: location, + }, + }, + ParseError::IllegalState(_location) => CLIError::InternalCompilerError { + src: source.to_string(), + }, + _ => { + todo!("Not yet handled {:?}", err); + } + } + } +} diff --git a/partiql-cli/src/bin/ion.sublime-syntax b/partiql-cli/src/ion.sublime-syntax similarity index 100% rename from partiql-cli/src/bin/ion.sublime-syntax rename to partiql-cli/src/ion.sublime-syntax diff --git a/partiql-cli/src/lib.rs b/partiql-cli/src/lib.rs new file mode 100644 index 00000000..287e5e00 --- /dev/null +++ b/partiql-cli/src/lib.rs @@ -0,0 +1,7 @@ +pub mod args; + +pub mod error; +pub mod repl; + +#[cfg(feature = "visualize")] +pub mod visualize; diff --git a/partiql-cli/src/main.rs b/partiql-cli/src/main.rs new file mode 100644 index 00000000..69c7d27b --- /dev/null +++ b/partiql-cli/src/main.rs @@ -0,0 +1,43 @@ +#![deny(rustdoc::broken_intra_doc_links)] + +use clap::Parser; +use partiql_cli::{args, repl}; + +use partiql_parser::Parsed; + +#[allow(dead_code)] +fn parse(query: &str) -> miette::Result { + let res = partiql_parser::Parser::default().parse(query); + //TODO + Ok(res.expect("parse failure")) +} + +fn main() -> miette::Result<()> { + let args = args::Args::parse(); + + match &args.command { + args::Commands::Repl => repl::repl(), + + #[cfg(feature = "visualize")] + args::Commands::Ast { format, query } => { + use partiql_cli::args::Format; + use partiql_cli::visualize::render::{display, to_dot, to_json, to_png, to_svg}; + use std::io::Write; + + let parsed = parse(&query)?; + match format { + Format::Json => println!("{}", to_json(&parsed.ast)), + Format::Dot => println!("{}", to_dot(&parsed.ast)), + Format::Svg => println!("{}", to_svg(&parsed.ast)), + Format::Png => { + std::io::stdout() + .write(&to_png(&parsed.ast)) + .expect("png write"); + } + Format::Display => display(&parsed.ast), + } + + Ok(()) + } + } +} diff --git a/partiql-cli/src/bin/partiql.sublime-syntax b/partiql-cli/src/partiql.sublime-syntax similarity index 100% rename from partiql-cli/src/bin/partiql.sublime-syntax rename to partiql-cli/src/partiql.sublime-syntax diff --git a/partiql-cli/src/bin/partiql-cli.rs b/partiql-cli/src/repl.rs similarity index 65% rename from partiql-cli/src/bin/partiql-cli.rs rename to partiql-cli/src/repl.rs index 13084878..6e329cdc 100644 --- a/partiql-cli/src/bin/partiql-cli.rs +++ b/partiql-cli/src/repl.rs @@ -9,7 +9,7 @@ use rustyline::hint::{Hinter, HistoryHinter}; use rustyline::validate::{ValidationContext, ValidationResult, Validator}; use rustyline::{ColorMode, Context, Helper}; use std::borrow::Cow; -use std::fs::{File, OpenOptions}; +use std::fs::OpenOptions; use std::path::Path; @@ -18,10 +18,10 @@ use syntect::highlighting::{Style, ThemeSet}; use syntect::parsing::{SyntaxDefinition, SyntaxSet, SyntaxSetBuilder}; use syntect::util::as_24_bit_terminal_escaped; -use miette::{Diagnostic, LabeledSpan, Report, SourceCode}; +use miette::Report; use owo_colors::OwoColorize; -use partiql_source_map::location::{BytePosition, Location}; -use thiserror::Error; + +use crate::error::CLIError; static ION_SYNTAX: &str = include_str!("ion.sublime-syntax"); static PARTIQL_SYNTAX: &str = include_str!("partiql.sublime-syntax"); @@ -83,6 +83,7 @@ impl Hinter for PartiqlHelper { hinter.hint(line, pos, ctx) } } + impl Highlighter for PartiqlHelper { fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> { let syntax = self @@ -106,91 +107,27 @@ impl Highlighter for PartiqlHelper { } } -#[derive(Debug, Error)] -pub enum CLIError { - #[error("PartiQL syntax error:")] - SyntaxError { - src: String, - msg: String, - loc: Location, - }, - // TODO add github issue link - #[error("Internal Compiler Error - please report this.")] - InternalCompilerError { src: String }, -} - -impl Diagnostic for CLIError { - fn source_code(&self) -> Option<&dyn SourceCode> { - match self { - CLIError::SyntaxError { src, .. } => Some(src), - CLIError::InternalCompilerError { src, .. } => Some(src), - } - } - - fn labels(&self) -> Option + '_>> { - match self { - CLIError::SyntaxError { msg, loc, .. } => { - Some(Box::new(std::iter::once(LabeledSpan::new( - Some(msg.to_string()), - loc.start.0 .0 as usize, - loc.end.0 .0 as usize - loc.start.0 .0 as usize, - )))) - } - CLIError::InternalCompilerError { .. } => None, - } - } -} - -impl CLIError { - pub fn from_parser_error(err: ParseError, source: &str) -> CLIError { - match err { - ParseError::SyntaxError(partiql_source_map::location::Located { inner, location }) => { - CLIError::SyntaxError { - src: source.to_string(), - msg: format!("Syntax error `{}`", inner), - loc: location, - } - } - ParseError::UnexpectedToken(partiql_source_map::location::Located { - inner, - location, - }) => CLIError::SyntaxError { - src: source.to_string(), - msg: format!("Unexpected token `{}`", inner.token), - loc: location, - }, - ParseError::LexicalError(partiql_source_map::location::Located { inner, location }) => { - CLIError::SyntaxError { - src: source.to_string(), - msg: format!("Lexical error `{}`", inner), - loc: location, - } - } - ParseError::Unknown(location) => CLIError::SyntaxError { - src: source.to_string(), - msg: "Unknown parser error".to_string(), - loc: Location { - start: location, - end: location, - }, - }, - ParseError::IllegalState(_location) => CLIError::InternalCompilerError { - src: source.to_string(), - }, - _ => { - todo!("Not yet handled {:?}", err); - } - } - } -} - impl Validator for PartiqlHelper { fn validate(&self, ctx: &mut ValidationContext) -> rustyline::Result { + // TODO remove this command parsing hack do something better + let mut source = ctx.input(); + let flag_display = source.starts_with("\\ast"); + if flag_display { + source = &source[4..]; + } + let parser = partiql_parser::Parser::default(); - let source = ctx.input(); let result = parser.parse(source); match result { - Ok(_) => Ok(ValidationResult::Valid(None)), + Ok(_parsed) => { + #[cfg(feature = "visualize")] + if flag_display { + use crate::visualize::render::display; + display(&_parsed.ast); + } + + Ok(ValidationResult::Valid(None)) + } Err(e) => { if e.errors .iter() @@ -214,7 +151,7 @@ impl Validator for PartiqlHelper { } } -fn main() -> miette::Result<()> { +pub fn repl() -> miette::Result<()> { let mut rl = rustyline::Editor::::new(); rl.set_color_mode(ColorMode::Forced); rl.set_helper(Some( @@ -252,9 +189,3 @@ fn main() -> miette::Result<()> { Ok(()) } - -#[cfg(test)] -mod tests { - #[test] - fn todo() {} -} diff --git a/partiql-cli/src/visualize/ast_to_dot.rs b/partiql-cli/src/visualize/ast_to_dot.rs new file mode 100644 index 00000000..0f408627 --- /dev/null +++ b/partiql-cli/src/visualize/ast_to_dot.rs @@ -0,0 +1,589 @@ +use partiql_ast::ast; +use std::fmt::Display; + +use dot_writer::{Attributes, DotWriter, Node, NodeId, Scope, Shape}; + +/* +subgraph cluster_legend { + rank = same; + variable[shape=Mdiamond] + literal[shape=rect] + "node"[shape=ellipse] +} + */ + +trait ScopeExt<'d, 'w> { + fn node_auto_labelled(&mut self, lbl: &str) -> Node<'_, 'w>; + fn cluster_auto_labelled(&mut self, lbl: &str) -> Scope<'_, 'w>; + fn with_cluster(&mut self, lbl: &str, func: F) -> R + where + F: FnMut(Scope<'_, 'w>) -> R; +} + +impl<'d, 'w> ScopeExt<'d, 'w> for Scope<'d, 'w> { + #[inline] + fn node_auto_labelled(&mut self, lbl: &str) -> Node<'_, 'w> { + let mut node = self.node_auto(); + node.set_label(lbl); + node + } + + fn cluster_auto_labelled(&mut self, lbl: &str) -> Scope<'_, 'w> { + let mut cluster = self.cluster(); + cluster.set("label", lbl, lbl.contains(" ")); + cluster + } + + fn with_cluster(&mut self, lbl: &str, mut func: F) -> R + where + F: FnMut(Scope<'_, 'w>) -> R, + { + let cluster = self.cluster_auto_labelled(lbl); + func(cluster) + } +} + +trait ChildEdgeExt { + fn edges(self, out: &mut Scope, from: &NodeId, lbl: &str) -> Targets; +} + +impl ChildEdgeExt for Targets { + fn edges(self, out: &mut Scope, from: &NodeId, lbl: &str) -> Targets { + for target in &self { + out.edge(&from, &target).attributes().set_label(lbl); + } + self + } +} + +type Targets = Vec; + +pub trait ToDotGraph { + fn to_graph(self, ast: &T) -> String; +} + +pub struct AstToDot {} + +impl Default for AstToDot { + fn default() -> Self { + AstToDot {} + } +} + +const BG_COLOR: &'static str = "\"#002b3600\""; +const FG_COLOR: &'static str = "\"#839496\""; + +impl ToDotGraph for AstToDot +where + AstToDot: ToDot, +{ + fn to_graph(mut self, ast: &T) -> String { + let mut output_bytes = Vec::new(); + + { + let mut writer = DotWriter::from(&mut output_bytes); + writer.set_pretty_print(true); + let mut digraph = writer.digraph(); + digraph + .graph_attributes() + .set_rank_direction(dot_writer::RankDirection::TopBottom) + .set("rankdir", "0.05", false) + .set("bgcolor", BG_COLOR, false) + .set("fontcolor", FG_COLOR, false) + .set("pencolor", FG_COLOR, false); + digraph.node_attributes().set("color", FG_COLOR, false).set( + "fontcolor", + FG_COLOR, + false, + ); + digraph.edge_attributes().set("color", FG_COLOR, false).set( + "fontcolor", + FG_COLOR, + false, + ); + + self.to_dot(&mut digraph, ast); + } + + return String::from_utf8(output_bytes).expect("invalid utf8"); + } +} + +trait ToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &T) -> Targets; +} + +impl ToDot> for AstToDot +where + AstToDot: ToDot, +{ + fn to_dot(&mut self, out: &mut Scope, ast: &Box) -> Targets { + self.to_dot(out, &**ast) + } +} + +impl ToDot> for AstToDot +where + AstToDot: ToDot, +{ + fn to_dot(&mut self, out: &mut Scope, asts: &Vec) -> Targets { + let mut res = Vec::with_capacity(asts.len()); + for ast in asts { + res.extend(self.to_dot(out, &ast)); + } + res + } +} + +impl ToDot> for AstToDot +where + AstToDot: ToDot, +{ + fn to_dot(&mut self, out: &mut Scope, ast: &Option) -> Targets { + match ast { + None => vec![], + Some(ast) => self.to_dot(out, &ast), + } + } +} + +impl ToDot> for AstToDot +where + AstToDot: ToDot, +{ + fn to_dot(&mut self, out: &mut Scope, ast: &ast::AstNode) -> Targets { + self.to_dot(out, &ast.node) + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Expr) -> Targets { + let mut expr_subgraph = out.subgraph(); + + use ast::ExprKind; + match &ast.kind { + ExprKind::Lit(l) => self.to_dot(&mut expr_subgraph, l), + ExprKind::VarRef(v) => self.to_dot(&mut expr_subgraph, v), + ExprKind::Param(_) => todo!(), + ExprKind::BinOp(bop) => self.to_dot(&mut expr_subgraph, bop), + ExprKind::UniOp(unop) => self.to_dot(&mut expr_subgraph, unop), + ExprKind::Like(like) => self.to_dot(&mut expr_subgraph, like), + ExprKind::Between(btwn) => self.to_dot(&mut expr_subgraph, btwn), + ExprKind::In(in_expr) => self.to_dot(&mut expr_subgraph, in_expr), + ExprKind::Case(_) => todo!(), + ExprKind::Struct(_) => todo!(), + ExprKind::Bag(_) => todo!(), + ExprKind::List(_) => todo!(), + ExprKind::Sexp(_) => todo!(), + ExprKind::Path(p) => self.to_dot(&mut expr_subgraph, p), + ExprKind::Call(c) => self.to_dot(&mut expr_subgraph, c), + ExprKind::CallAgg(c) => self.to_dot(&mut expr_subgraph, c), + ExprKind::Query(q) => self.to_dot(&mut expr_subgraph, q), + ExprKind::Error => todo!(), + } + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Lit) -> Targets { + use ast::Lit; + let lbl = match ast { + Lit::Null => "NULL".to_string(), + Lit::Missing => "MISSING".to_string(), + Lit::Int8Lit(l) => l.to_string(), + Lit::Int16Lit(l) => l.to_string(), + Lit::Int32Lit(l) => l.to_string(), + Lit::Int64Lit(l) => l.to_string(), + Lit::DecimalLit(l) => l.to_string(), + Lit::NumericLit(l) => l.to_string(), + Lit::RealLit(l) => l.to_string(), + Lit::FloatLit(l) => l.to_string(), + Lit::DoubleLit(l) => l.to_string(), + Lit::BoolLit(l) => (if *l { "TRUE" } else { "FALSE" }).to_string(), + Lit::IonStringLit(l) => format!("`{}`", l), + Lit::CharStringLit(l) => format!("'{}'", l), + Lit::NationalCharStringLit(l) => format!("'{}'", l), + Lit::BitStringLit(l) => format!("b'{}'", l), + Lit::HexStringLit(l) => format!("x'{}'", l), + Lit::DateTimeLit(l) => match l { + ast::DateTimeLit::DateLit(d) => format!("DATE '{}'", d), + ast::DateTimeLit::TimeLit(t) => format!("TIME '{}'", t), + ast::DateTimeLit::TimestampLit(ts) => format!("TIMESTAMP '{}'", ts), + }, + Lit::CollectionLit(l) => match l { + ast::CollectionLit::ArrayLit(al) => format!("[{}]", al), + ast::CollectionLit::BagLit(bl) => format!("<<{}>>", bl), + }, + }; + + let mut node = out.node_auto(); + node.set_label(&lbl).set_shape(Shape::Rectangle); + + vec![node.id()] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::BinOp) -> Targets { + use ast::BinOpKind; + let lbl = match ast.kind { + BinOpKind::Add => "+", + BinOpKind::Div => "/", + BinOpKind::Exp => "^", + BinOpKind::Mod => "%", + BinOpKind::Mul => "*", + BinOpKind::Neg => "-", + BinOpKind::And => "AND", + BinOpKind::Or => "OR", + BinOpKind::Concat => "||", + BinOpKind::Eq => "=", + BinOpKind::Gt => ">", + BinOpKind::Gte => ">=", + BinOpKind::Lt => "<", + BinOpKind::Lte => "<=", + BinOpKind::Ne => "<>", + BinOpKind::Is => "IS", + }; + let id = out.node_auto_labelled(lbl).id(); + + self.to_dot(out, &ast.lhs).edges(out, &id, ""); + self.to_dot(out, &ast.rhs).edges(out, &id, ""); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::UniOp) -> Targets { + use ast::UniOpKind; + let lbl = match ast.kind { + UniOpKind::Pos => "+", + UniOpKind::Neg => "-", + UniOpKind::Not => "NOT", + }; + let id = out.node_auto_labelled(lbl).id(); + + self.to_dot(out, &ast.expr).edges(out, &id, ""); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Like) -> Targets { + let id = out.node_auto_labelled("LIKE").id(); + + self.to_dot(out, &ast.value).edges(out, &id, "value"); + self.to_dot(out, &ast.pattern).edges(out, &id, "pattern"); + self.to_dot(out, &ast.escape).edges(out, &id, "escape"); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Between) -> Targets { + let id = out.node_auto_labelled("BETWEEN").id(); + + self.to_dot(out, &ast.value).edges(out, &id, "value"); + self.to_dot(out, &ast.from).edges(out, &id, "from"); + self.to_dot(out, &ast.to).edges(out, &id, "to"); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::In) -> Targets { + let id = out.node_auto_labelled("IN").id(); + + self.to_dot(out, &ast.lhs).edges(out, &id, ""); + self.to_dot(out, &ast.rhs).edges(out, &id, ""); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Query) -> Targets { + let id = out.node_auto_labelled("Query").id(); + + self.to_dot(out, &ast.set).edges(out, &id, ""); + self.to_dot(out, &ast.order_by).edges(out, &id, "order_by"); + self.to_dot(out, &ast.limit).edges(out, &id, "limit"); + self.to_dot(out, &ast.offset).edges(out, &id, "offset"); + + vec![id] + } +} +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::QuerySet) -> Targets { + use ast::QuerySet; + match &ast { + QuerySet::SetOp(_) => todo!(), + QuerySet::Select(select) => self.to_dot(out, select), + QuerySet::Expr(e) => self.to_dot(out, e), + QuerySet::Values(_) => todo!(), + } + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Select) -> Targets { + let id = out.node_auto_labelled("Select").id(); + + out.with_cluster("PROJECT", |mut cl| self.to_dot(&mut cl, &ast.project)) + .edges(out, &id, ""); + out.with_cluster("FROM", |mut cl| self.to_dot(&mut cl, &ast.from)) + .edges(out, &id, ""); + out.with_cluster("FROM LET", |mut cl| self.to_dot(&mut cl, &ast.from_let)) + .edges(out, &id, ""); + out.with_cluster("WHERE", |mut cl| self.to_dot(&mut cl, &ast.where_clause)) + .edges(out, &id, ""); + out.with_cluster("GROUP BY", |mut cl| self.to_dot(&mut cl, &ast.group_by)) + .edges(out, &id, ""); + out.with_cluster("HAVING", |mut cl| self.to_dot(&mut cl, &ast.having)) + .edges(out, &id, ""); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Projection) -> Targets { + let lbl = match &ast.setq { + Some(ast::SetQuantifier::Distinct) => "Projection | Distinct", + _ => "Projection | All", + }; + let id = out.node_auto_labelled(lbl).id(); + + use ast::ProjectionKind; + let children = { + let mut expr_subgraph = out.subgraph(); + + match &ast.kind { + ProjectionKind::ProjectStar => vec![expr_subgraph.node_auto_labelled("*").id()], + ProjectionKind::ProjectList(items) => { + let mut list = vec![]; + for item in items { + list.extend(self.to_dot(&mut expr_subgraph, item)); + } + list + } + ProjectionKind::ProjectPivot { .. } => todo!(), + ProjectionKind::ProjectValue(_) => todo!(), + } + }; + + children.edges(out, &id, ""); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::ProjectItem) -> Targets { + match ast { + ast::ProjectItem::ProjectAll(all) => { + let id = out.node_auto_labelled("ProjectAll").id(); + self.to_dot(out, &all.expr).edges(out, &id, ""); + vec![id] + } + ast::ProjectItem::ProjectExpr(expr) => { + let id = out.node_auto_labelled("ProjectExpr").id(); + self.to_dot(out, &expr.expr).edges(out, &id, ""); + self.to_dot(out, &expr.as_alias).edges(out, &id, "as"); + vec![id] + } + } + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::SymbolPrimitive) -> Targets { + use ast::CaseSensitivity; + let case = ast.case.clone().unwrap_or(CaseSensitivity::CaseInsensitive); + let lbl = match case { + CaseSensitivity::CaseSensitive => format!("'{}'", ast.value), + CaseSensitivity::CaseInsensitive => format!("{}", ast.value), + }; + let id = out.node_auto_labelled(&lbl).id(); + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::VarRef) -> Targets { + use ast::CaseSensitivity; + let case = ast + .name + .case + .clone() + .unwrap_or(CaseSensitivity::CaseInsensitive); + let prefix = match &ast.qualifier { + ast::ScopeQualifier::Unqualified => "", + ast::ScopeQualifier::Qualified => "@", + }; + let lbl = match case { + CaseSensitivity::CaseSensitive => format!("{}'{}'", prefix, ast.name.value), + CaseSensitivity::CaseInsensitive => format!("{}{}", prefix, ast.name.value), + }; + let id = out.node_auto_labelled(&lbl).id(); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, _out: &mut Scope, _ast: &ast::OrderByExpr) -> Targets { + todo!("OrderByExpr"); + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, _out: &mut Scope, _ast: &ast::GroupByExpr) -> Targets { + todo!("GroupByExpr"); + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::FromClause) -> Targets { + match &ast { + ast::FromClause::FromLet(fl) => self.to_dot(out, fl), + ast::FromClause::Join(j) => self.to_dot(out, j), + } + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::FromLet) -> Targets { + let lbl = match &ast.kind { + ast::FromLetKind::Scan => "Scan", + ast::FromLetKind::Unpivot => "Unpivot", + }; + let id = out.node_auto_labelled(lbl).id(); + + self.to_dot(out, &ast.expr).edges(out, &id, ""); + self.to_dot(out, &ast.as_alias).edges(out, &id, "as"); + self.to_dot(out, &ast.at_alias).edges(out, &id, "at"); + self.to_dot(out, &ast.by_alias).edges(out, &id, "by"); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Join) -> Targets { + let lbl = match &ast.kind { + ast::JoinKind::Inner => "Inner Join", + ast::JoinKind::Left => "Left Join", + ast::JoinKind::Right => "Right Join", + ast::JoinKind::Full => "Full Join", + ast::JoinKind::Cross => "Cross Join", + }; + let id = out.node_auto_labelled(lbl).id(); + + self.to_dot(out, &ast.left).edges(out, &id, "left"); + self.to_dot(out, &ast.right).edges(out, &id, "right"); + self.to_dot(out, &ast.predicate) + .edges(out, &id, "predicate"); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::JoinSpec) -> Targets { + match &ast { + ast::JoinSpec::On(fl) => { + let id = out.node_auto_labelled("On").id(); + self.to_dot(out, fl).edges(out, &id, ""); + vec![id] + } + ast::JoinSpec::Using(j) => { + let id = out.node_auto_labelled("Using").id(); + self.to_dot(out, j).edges(out, &id, ""); + vec![id] + } + ast::JoinSpec::Natural => vec![out.node_auto_labelled("Natural").id()], + } + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Call) -> Targets { + let id = out.node_auto_labelled("Call").id(); + + self.to_dot(out, &ast.func_name).edges(out, &id, "name"); + self.to_dot(out, &ast.args).edges(out, &id, "args"); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::CallArg) -> Targets { + match ast { + ast::CallArg::Star() => vec![out.node_auto_labelled("*").id()], + ast::CallArg::Positional(e) => self.to_dot(out, e), + ast::CallArg::Named { name, value } => { + let id = out.node_auto_labelled("Named").id(); + self.to_dot(out, name).edges(out, &id, "name"); + self.to_dot(out, value).edges(out, &id, "value"); + vec![id] + } + } + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::CallAgg) -> Targets { + let lbl = match &ast.setq { + Some(ast::SetQuantifier::Distinct) => "CallAgg | Distinct", + _ => "CallAgg | All", + }; + let id = out.node_auto_labelled(lbl).id(); + + self.to_dot(out, &ast.func_name).edges(out, &id, "name"); + self.to_dot(out, &ast.args).edges(out, &id, "args"); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::Path) -> Targets { + let id = out.node_auto_labelled("Path").id(); + + self.to_dot(out, &ast.root).edges(out, &id, "root"); + self.to_dot(out, &ast.steps).edges(out, &id, "steps"); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::PathStep) -> Targets { + match &ast { + ast::PathStep::PathExpr(e) => self.to_dot(out, e), + ast::PathStep::PathWildCard => vec![out.node_auto_labelled("*").id()], + ast::PathStep::PathUnpivot => vec![out.node_auto_labelled("Unpivot").id()], + } + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, out: &mut Scope, ast: &ast::PathExpr) -> Targets { + let id = out.node_auto_labelled("PathExpr").id(); + + self.to_dot(out, &ast.index).edges(out, &id, "index"); + + vec![id] + } +} + +impl ToDot for AstToDot { + fn to_dot(&mut self, _out: &mut Scope, _ast: &ast::Let) -> Targets { + todo!("Let"); + } +} diff --git a/partiql-cli/src/visualize/mod.rs b/partiql-cli/src/visualize/mod.rs new file mode 100644 index 00000000..4037179d --- /dev/null +++ b/partiql-cli/src/visualize/mod.rs @@ -0,0 +1,2 @@ +pub mod ast_to_dot; +pub mod render; diff --git a/partiql-cli/src/visualize/render.rs b/partiql-cli/src/visualize/render.rs new file mode 100644 index 00000000..2371608d --- /dev/null +++ b/partiql-cli/src/visualize/render.rs @@ -0,0 +1,132 @@ +use crate::visualize::ast_to_dot::{AstToDot, ToDotGraph}; + +use std::convert::AsRef; +use std::os::raw::c_char; +use std::slice; +use strum::AsRefStr; + +use graphviz_sys as gv; +use partiql_ast::ast; + +/// Convert an AST into JSON +#[inline] +pub fn to_json(ast: &Box) -> String { + serde_json::to_string_pretty(&ast).expect("json print") +} + +/// Graphviz output formats +#[derive(AsRefStr, Debug, Copy, Clone)] +#[strum(serialize_all = "lowercase")] +#[non_exhaustive] +pub enum GraphVizFormat { + /// Pretty-print + Canon, + /// Pretty-print; internal alias for graphviz's `canon` + /// #[strum(serialize = "cannon")] + PrettyPrint, + /// Attributed dot + Dot, + /// Extended dot + XDot, + /// Svg + Svg, + /// Png + Png, +} + +/// FFI to graphviz-sys to convert a dot-formatted graph into the specified format. +fn gv_render(format: GraphVizFormat, graph_str: String) -> Vec { + let c_graph_str = std::ffi::CString::new(graph_str).expect("cstring new failed"); + let c_dot = std::ffi::CString::new("dot").expect("cstring new failed"); + let c_format = std::ffi::CString::new(format.as_ref()).expect("cstring new failed"); + + unsafe { + let gvc = gv::gvContext(); + // TODO gvParseArgs to pass 'theme' colors, etc? + // See section 4 of https://www.graphviz.org/pdf/libguide.pdf + // See `dot --help` + let g = gv::agmemread(c_graph_str.as_ptr()); + + gv::gvLayout(gvc, g, c_dot.as_ptr()); + + let mut buffer_ptr: *mut std::os::raw::c_char = std::ptr::null_mut(); + let mut length = 0 as std::os::raw::c_uint; + gv::gvRenderData(gvc, g, c_format.as_ptr(), &mut buffer_ptr, &mut length); + let c_bytes = slice::from_raw_parts_mut(buffer_ptr, length as usize); + + let bytes = std::mem::transmute::<&mut [c_char], &[u8]>(c_bytes); + let out = Vec::from(bytes); + + gv::gvFreeRenderData(buffer_ptr); + gv::gvFreeLayout(gvc, g); + gv::agclose(g); + gv::gvFreeContext(gvc); + + out + } +} + +/// Convert an AST into a graphviz dot-formatted string +#[inline] +fn ast_to_dot(ast: &Box) -> String { + AstToDot::default().to_graph(ast) +} + +/// FFI to graphviz-sys to convert a dot-formatted graph into the specified text format. +#[inline] +fn render_to_string(format: GraphVizFormat, ast: &Box) -> String { + String::from_utf8(gv_render(format, ast_to_dot(ast))).expect("valid utf8") +} + +/// Convert an AST into an attributed dot graph. +#[inline] +pub fn to_dot(ast: &Box) -> String { + render_to_string(GraphVizFormat::Dot, &ast) +} + +/// Convert an AST into a pretty-printed dot graph. +#[inline] +pub fn to_pretty_dot(ast: &Box) -> String { + render_to_string(GraphVizFormat::PrettyPrint, &ast) +} + +/// Convert an AST into a graphviz svg. +#[inline] +pub fn to_svg(ast: &Box) -> String { + render_to_string(GraphVizFormat::Svg, &ast) +} + +/// Convert an AST into a graphviz svg and render it to png. +pub fn to_png(ast: &Box) -> Vec { + let svg_data = to_svg(ast); + + let mut opt = usvg::Options::default(); + opt.fontdb.load_system_fonts(); + + let rtree = usvg::Tree::from_data(svg_data.as_bytes(), &opt.to_ref()).unwrap(); + let pixmap_size = rtree.svg_node().size.to_screen_size(); + let mut pixmap = tiny_skia::Pixmap::new(pixmap_size.width(), pixmap_size.height()).unwrap(); + resvg::render( + &rtree, + usvg::FitTo::Original, + tiny_skia::Transform::default(), + pixmap.as_mut(), + ) + .unwrap(); + pixmap.encode_png().expect("png encoding failed") +} + +/// Convert an AST into a graphviz svg and render it to png, then display in the console. +pub fn display(ast: &Box) { + let png = to_png(ast); + + let conf = viuer::Config { + absolute_offset: false, + transparent: true, + use_sixel: false, + ..Default::default() + }; + + let img = image::load_from_memory(&png).expect("png loading failed."); + viuer::print(&img, &conf).expect("Image printing failed."); +} From 7cef2ba7cbded8b081769b7475ba07785ed3f212 Mon Sep 17 00:00:00 2001 From: Josh Pschorr Date: Tue, 26 Jul 2022 13:02:01 -0700 Subject: [PATCH 2/2] Add visualization stuff to README + minor cleanup --- partiql-cli/Cargo.toml | 36 +++++++++++++++++----------------- partiql-cli/README.md | 42 +++++++++++++++++++++++++++++++++------- partiql-cli/src/error.rs | 29 +++++++++++++++++++++++---- partiql-cli/src/main.rs | 9 +++++---- partiql-cli/src/repl.rs | 16 +++++---------- 5 files changed, 88 insertions(+), 44 deletions(-) diff --git a/partiql-cli/Cargo.toml b/partiql-cli/Cargo.toml index 19ebbf24..dcdbc612 100644 --- a/partiql-cli/Cargo.toml +++ b/partiql-cli/Cargo.toml @@ -33,33 +33,33 @@ partiql-source-map = { path = "../partiql-source-map" } partiql-ast = { path = "../partiql-ast" } -rustyline = "9.1.2" -syntect = "5.0" -owo-colors = "3.4.0" -supports-color = "1.3.0" -supports-unicode = "1.0.2" -supports-hyperlinks = "1.2.0" -termbg = "0.4.1" -shellexpand = "2.1.0" +rustyline = "10.*" +syntect = "5.*" +owo-colors = "3.*" +supports-color = "1.*" +supports-unicode = "1.*" +supports-hyperlinks = "1.*" +termbg = "0.4.*" +shellexpand = "2.*" -thiserror = "1.0.31" -miette = { version ="4.7.1", features = ["fancy"] } -clap = { version = "3.2.13", features = ["derive"] } +thiserror = "1.*" +miette = { version ="5.*", features = ["fancy"] } +clap = { version = "3.*", features = ["derive"] } # serde serde = { version ="1.*", features = ["derive"], optional = true } serde_json = { version ="1.*", optional = true } ### Dependencies for the `render` feature -viuer = { version ="0.6.1", features = ["sixel"], optional = true } -image = { version ="0.24.2", optional = true } +viuer = { version ="0.6.*", features = ["sixel"], optional = true } +image = { version ="0.24.*", optional = true } graphviz-sys = { version ="0.1.3", optional = true } -resvg = { version ="0.23.0", optional = true } -usvg = { version ="0.23.0", optional = true } -tiny-skia = { version ="0.6.6", optional = true } -strum = { version ="0.24.1", features = ["derive"], optional = true } -dot-writer = { version = "~0.1.2", optional = true } +resvg = { version ="0.23.*", optional = true } +usvg = { version ="0.23.*", optional = true } +tiny-skia = { version ="0.6.*", optional = true } +strum = { version ="0.24.*", features = ["derive"], optional = true } +dot-writer = { version = "0.1.*", optional = true } diff --git a/partiql-cli/README.md b/partiql-cli/README.md index db733d44..1694097e 100644 --- a/partiql-cli/README.md +++ b/partiql-cli/README.md @@ -1,11 +1,29 @@ -# PartiQL Rust REPL +# PartiQL Rust CLI +PoC for a CLI & REPL. It should be considered experimental, subject to change, etc. -PoC for a REPL. It should be considered experimental, subject to change, etc. - -In its current state, it largely exists to test parser interface & types from the perspective of an external application. +In its current state, it largely exists to test parser interface & types from the perspective of an external application. Probably the the mietter::Diagnostic stuff should be refactored and moved to the main parser crate. -The REPL currently accepts no commands, assuming any/all input is a PartiQL query, which it will attempt to parse. Parse errors are pretty printed to the output. +## CLI Commands + +- **`help`** : print the CLI's help message and supported commands +- **`repl`** : launches the [REPL](##REPL) +- **`ast -T ""`**: outputs a rendered version of the parsed AST ([see Visualization](##Visualizations)): + - **``**: + - **`json`** : pretty-print to stdout in a json dump + - **`dot`** : pretty-print to stdout in [Graphviz][Graphviz] [dot][GvDot] format + - **`svg`** : print to stdout a [Graphviz][Graphviz] rendered svg xml document + - **`png`** : print to stdout a [Graphviz][Graphviz] rendered png bitmap + - **`display`** : display a [Graphviz][Graphviz] rendered png bitmap directly in supported terminals + - **`query`** : the PartiQL query text + +## REPL + +The REPL currently assumes most of the input line is a PartiQL query, which it will attempt to parse. +- For an invalid query, errors are pretty printed to the output. +- For a valid query, + - with no prefix, `Parse OK!` is printed to the output + - if prefixed by `\ast`, a rendered AST tree image is printed to the output ([see Visualization](##Visualizations)) Features: - Syntax highlighting of query input @@ -13,11 +31,21 @@ Features: - Readling/editing - `CTRL-D`/`CTRL-C` to quit. +# Visualizations + +In order to use any of the [Graphviz][Graphviz]-based visualizations, you will need the graphviz libraries +installed on your machine (e.g. `brew install graphviz` or similar). + # TODO +See [REPL-tagged issues](https://github.com/partiql/partiql-lang-rust/issues?q=is%3Aissue+is%3Aopen+%5BREPL%5D) + - Use central location for syntax files rather than embedded in this crate -- Add github issue link for Internal Compiler Errors - Better interaction model - commands - more robust editing - - etc. \ No newline at end of file + - etc. + + +[Graphviz]: https://graphviz.org/ +[GvDot]: https://graphviz.org/doc/info/lang.html \ No newline at end of file diff --git a/partiql-cli/src/error.rs b/partiql-cli/src/error.rs index 69640e20..2ae03c13 100644 --- a/partiql-cli/src/error.rs +++ b/partiql-cli/src/error.rs @@ -1,8 +1,30 @@ use miette::{Diagnostic, LabeledSpan, SourceCode}; -use partiql_parser::ParseError; +use partiql_parser::{ParseError, ParserError}; use partiql_source_map::location::{BytePosition, Location}; + use thiserror::Error; +#[derive(Debug, Error, Diagnostic)] +#[error("Error for query `{query}`")] +pub struct CLIErrors { + query: String, + #[related] + related: Vec, +} + +impl CLIErrors { + pub fn from_parser_error(err: ParserError) -> Self { + let query = err.text.to_string(); + + let related = err + .errors + .into_iter() + .map(|e| CLIError::from_parse_error(e, &query)) + .collect(); + CLIErrors { query, related } + } +} + #[derive(Debug, Error)] pub enum CLIError { #[error("PartiQL syntax error:")] @@ -11,8 +33,7 @@ pub enum CLIError { msg: String, loc: Location, }, - // TODO add github issue link - #[error("Internal Compiler Error - please report this.")] + #[error("Internal Compiler Error - please report this (https://github.com/partiql/partiql-lang-rust/issues).")] InternalCompilerError { src: String }, } @@ -39,7 +60,7 @@ impl Diagnostic for CLIError { } impl CLIError { - pub fn from_parser_error(err: ParseError, source: &str) -> CLIError { + pub fn from_parse_error(err: ParseError, source: &str) -> Self { match err { ParseError::SyntaxError(partiql_source_map::location::Located { inner, location }) => { CLIError::SyntaxError { diff --git a/partiql-cli/src/main.rs b/partiql-cli/src/main.rs index 69c7d27b..bf0aada7 100644 --- a/partiql-cli/src/main.rs +++ b/partiql-cli/src/main.rs @@ -1,15 +1,16 @@ #![deny(rustdoc::broken_intra_doc_links)] use clap::Parser; +use partiql_cli::error::CLIErrors; use partiql_cli::{args, repl}; use partiql_parser::Parsed; #[allow(dead_code)] -fn parse(query: &str) -> miette::Result { - let res = partiql_parser::Parser::default().parse(query); - //TODO - Ok(res.expect("parse failure")) +fn parse(query: &str) -> Result { + partiql_parser::Parser::default() + .parse(query) + .map_err(CLIErrors::from_parser_error) } fn main() -> miette::Result<()> { diff --git a/partiql-cli/src/repl.rs b/partiql-cli/src/repl.rs index 6e329cdc..e667adbb 100644 --- a/partiql-cli/src/repl.rs +++ b/partiql-cli/src/repl.rs @@ -18,10 +18,10 @@ use syntect::highlighting::{Style, ThemeSet}; use syntect::parsing::{SyntaxDefinition, SyntaxSet, SyntaxSetBuilder}; use syntect::util::as_24_bit_terminal_escaped; -use miette::Report; +use miette::{IntoDiagnostic, Report}; use owo_colors::OwoColorize; -use crate::error::CLIError; +use crate::error::CLIErrors; static ION_SYNTAX: &str = include_str!("ion.sublime-syntax"); static PARTIQL_SYNTAX: &str = include_str!("partiql.sublime-syntax"); @@ -137,14 +137,8 @@ impl Validator for PartiqlHelper { // TODO we should probably do something more ergonomic. Perhaps require a `;` or two newlines to end? Ok(ValidationResult::Incomplete) } else { - let err_msg = e - .errors - .into_iter() - .map(|e| CLIError::from_parser_error(e, source)) - .map(|e| format!("{:?}", Report::new(e))) - .collect::>() - .join("\n"); - Ok(ValidationResult::Invalid(Some(format!("\n\n{}", err_msg)))) + let err = Report::new(CLIErrors::from_parser_error(e)); + Ok(ValidationResult::Invalid(Some(format!("\n\n{:?}", err)))) } } } @@ -152,7 +146,7 @@ impl Validator for PartiqlHelper { } pub fn repl() -> miette::Result<()> { - let mut rl = rustyline::Editor::::new(); + let mut rl = rustyline::Editor::::new().into_diagnostic()?; rl.set_color_mode(ColorMode::Forced); rl.set_helper(Some( PartiqlHelper::new(PartiqlHelperConfig::infer()).unwrap(),