diff --git a/404.html b/404.html deleted file mode 100644 index c472b4e..0000000 --- a/404.html +++ /dev/null @@ -1,24 +0,0 @@ ---- -layout: default ---- - - - -

404

- -

Page not found :(

The requested page could not be found.

diff --git a/Gemfile b/Gemfile deleted file mode 100644 index b4e47da..0000000 --- a/Gemfile +++ /dev/null @@ -1,32 +0,0 @@ -source "https://rubygems.org" - -# Hello! This is where you manage which Jekyll version is used to run. -# When you want to use a different version, change it below, save the -# file and run `bundle install`. Run Jekyll with `bundle exec`, like so: -# -# bundle exec jekyll serve -# -# This will help ensure the proper Jekyll version is running. -# Happy Jekylling! -#gem "jekyll", "~> 3.7.2" - -# This is the default theme for new Jekyll sites. You may change this to anything you like. -# gem "minima", "~> 2.0" - -# If you want to use GitHub Pages, remove the "gem "jekyll"" above and -# uncomment the line below. To upgrade, run `bundle update github-pages`. -gem "github-pages", group: :jekyll_plugins - -# If you have any plugins, put them here! -group :jekyll_plugins do - gem "jekyll-feed", "~> 0.6" -end - -gem 'jekyll-relative-links' - -# Windows does not include zoneinfo files, so bundle the tzinfo-data gem -#gem "tzinfo-data", platforms: [:mingw, :mswin, :x64_mingw, :jruby] - -# Performance-booster for watching directories on Windows -#gem "wdm", "~> 0.1.0" if Gem.win_platform? - diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index bdc4254..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,251 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - activesupport (4.2.11.1) - i18n (~> 0.7) - minitest (~> 5.1) - thread_safe (~> 0.3, >= 0.3.4) - tzinfo (~> 1.1) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) - coffee-script (2.4.1) - coffee-script-source - execjs - coffee-script-source (1.11.1) - colorator (1.1.0) - commonmarker (0.17.13) - ruby-enum (~> 0.5) - concurrent-ruby (1.1.5) - dnsruby (1.61.3) - addressable (~> 2.5) - em-websocket (0.5.1) - eventmachine (>= 0.12.9) - http_parser.rb (~> 0.6.0) - ethon (0.12.0) - ffi (>= 1.3.0) - eventmachine (1.2.7) - execjs (2.7.0) - faraday (0.17.0) - multipart-post (>= 1.2, < 3) - ffi (1.11.1) - forwardable-extended (2.6.0) - gemoji (3.0.1) - github-pages (201) - activesupport (= 4.2.11.1) - github-pages-health-check (= 1.16.1) - jekyll (= 3.8.5) - jekyll-avatar (= 0.6.0) - jekyll-coffeescript (= 1.1.1) - jekyll-commonmark-ghpages (= 0.1.6) - jekyll-default-layout (= 0.1.4) - jekyll-feed (= 0.11.0) - jekyll-gist (= 1.5.0) - jekyll-github-metadata (= 2.12.1) - jekyll-mentions (= 1.4.1) - jekyll-optional-front-matter (= 0.3.0) - jekyll-paginate (= 1.1.0) - jekyll-readme-index (= 0.2.0) - jekyll-redirect-from (= 0.14.0) - jekyll-relative-links (= 0.6.0) - jekyll-remote-theme (= 0.4.0) - jekyll-sass-converter (= 1.5.2) - jekyll-seo-tag (= 2.5.0) - jekyll-sitemap (= 1.2.0) - jekyll-swiss (= 0.4.0) - jekyll-theme-architect (= 0.1.1) - jekyll-theme-cayman (= 0.1.1) - jekyll-theme-dinky (= 0.1.1) - jekyll-theme-hacker (= 0.1.1) - jekyll-theme-leap-day (= 0.1.1) - jekyll-theme-merlot (= 0.1.1) - jekyll-theme-midnight (= 0.1.1) - jekyll-theme-minimal (= 0.1.1) - jekyll-theme-modernist (= 0.1.1) - jekyll-theme-primer (= 0.5.3) - jekyll-theme-slate (= 0.1.1) - jekyll-theme-tactile (= 0.1.1) - jekyll-theme-time-machine (= 0.1.1) - jekyll-titles-from-headings (= 0.5.1) - jemoji (= 0.10.2) - kramdown (= 1.17.0) - liquid (= 4.0.0) - listen (= 3.1.5) - mercenary (~> 0.3) - minima (= 2.5.0) - nokogiri (>= 1.10.4, < 2.0) - rouge (= 3.11.0) - terminal-table (~> 1.4) - github-pages-health-check (1.16.1) - addressable (~> 2.3) - dnsruby (~> 1.60) - octokit (~> 4.0) - public_suffix (~> 3.0) - typhoeus (~> 1.3) - html-pipeline (2.12.0) - activesupport (>= 2) - nokogiri (>= 1.4) - http_parser.rb (0.6.0) - i18n (0.9.5) - concurrent-ruby (~> 1.0) - jekyll (3.8.5) - addressable (~> 2.4) - colorator (~> 1.0) - em-websocket (~> 0.5) - i18n (~> 0.7) - jekyll-sass-converter (~> 1.0) - jekyll-watch (~> 2.0) - kramdown (~> 1.14) - liquid (~> 4.0) - mercenary (~> 0.3.3) - pathutil (~> 0.9) - rouge (>= 1.7, < 4) - safe_yaml (~> 1.0) - jekyll-avatar (0.6.0) - jekyll (~> 3.0) - jekyll-coffeescript (1.1.1) - coffee-script (~> 2.2) - coffee-script-source (~> 1.11.1) - jekyll-commonmark (1.3.1) - commonmarker (~> 0.14) - jekyll (>= 3.7, < 5.0) - jekyll-commonmark-ghpages (0.1.6) - commonmarker (~> 0.17.6) - jekyll-commonmark (~> 1.2) - rouge (>= 2.0, < 4.0) - jekyll-default-layout (0.1.4) - jekyll (~> 3.0) - jekyll-feed (0.11.0) - jekyll (~> 3.3) - jekyll-gist (1.5.0) - octokit (~> 4.2) - jekyll-github-metadata (2.12.1) - jekyll (~> 3.4) - octokit (~> 4.0, != 4.4.0) - jekyll-mentions (1.4.1) - html-pipeline (~> 2.3) - jekyll (~> 3.0) - jekyll-optional-front-matter (0.3.0) - jekyll (~> 3.0) - jekyll-paginate (1.1.0) - jekyll-readme-index (0.2.0) - jekyll (~> 3.0) - jekyll-redirect-from (0.14.0) - jekyll (~> 3.3) - jekyll-relative-links (0.6.0) - jekyll (~> 3.3) - jekyll-remote-theme (0.4.0) - addressable (~> 2.0) - jekyll (~> 3.5) - rubyzip (>= 1.2.1, < 3.0) - jekyll-sass-converter (1.5.2) - sass (~> 3.4) - jekyll-seo-tag (2.5.0) - jekyll (~> 3.3) - jekyll-sitemap (1.2.0) - jekyll (~> 3.3) - jekyll-swiss (0.4.0) - jekyll-theme-architect (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-cayman (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-dinky (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-hacker (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-leap-day (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-merlot (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-midnight (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-minimal (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-modernist (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-primer (0.5.3) - jekyll (~> 3.5) - jekyll-github-metadata (~> 2.9) - jekyll-seo-tag (~> 2.0) - jekyll-theme-slate (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-tactile (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-time-machine (0.1.1) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-titles-from-headings (0.5.1) - jekyll (~> 3.3) - jekyll-watch (2.2.1) - listen (~> 3.0) - jemoji (0.10.2) - gemoji (~> 3.0) - html-pipeline (~> 2.2) - jekyll (~> 3.0) - kramdown (1.17.0) - liquid (4.0.0) - listen (3.1.5) - rb-fsevent (~> 0.9, >= 0.9.4) - rb-inotify (~> 0.9, >= 0.9.7) - ruby_dep (~> 1.2) - mercenary (0.3.6) - mini_portile2 (2.4.0) - minima (2.5.0) - jekyll (~> 3.5) - jekyll-feed (~> 0.9) - jekyll-seo-tag (~> 2.1) - minitest (5.12.2) - multipart-post (2.1.1) - nokogiri (1.10.10) - mini_portile2 (~> 2.4.0) - octokit (4.14.0) - sawyer (~> 0.8.0, >= 0.5.3) - pathutil (0.16.2) - forwardable-extended (~> 2.6) - public_suffix (3.1.1) - rb-fsevent (0.10.3) - rb-inotify (0.10.0) - ffi (~> 1.0) - rouge (3.11.0) - ruby-enum (0.7.2) - i18n - ruby_dep (1.5.0) - rubyzip (2.0.0) - safe_yaml (1.0.5) - sass (3.7.4) - sass-listen (~> 4.0.0) - sass-listen (4.0.0) - rb-fsevent (~> 0.9, >= 0.9.4) - rb-inotify (~> 0.9, >= 0.9.7) - sawyer (0.8.2) - addressable (>= 2.3.5) - faraday (> 0.8, < 2.0) - terminal-table (1.8.0) - unicode-display_width (~> 1.1, >= 1.1.1) - thread_safe (0.3.6) - typhoeus (1.3.1) - ethon (>= 0.9.0) - tzinfo (1.2.5) - thread_safe (~> 0.1) - unicode-display_width (1.6.0) - -PLATFORMS - ruby - -DEPENDENCIES - github-pages - jekyll-feed (~> 0.6) - jekyll-relative-links - -BUNDLED WITH - 2.0.2 diff --git a/_config.yml b/_config.yml deleted file mode 100644 index 07b6844..0000000 --- a/_config.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Welcome to Jekyll! -# -# This config file is meant for settings that affect your whole blog, values -# which you are expected to set up once and rarely edit after that. If you find -# yourself editing this file very often, consider using Jekyll's data files -# feature for the data you need to update frequently. -# -# For technical reasons, this file is *NOT* reloaded automatically when you use -# 'bundle exec jekyll serve'. If you change this file, please restart the server process. - -# Site settings -# These are used to personalize your new site. If you look in the HTML files, -# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. -# You can create any custom variable you would like, and they will be accessible -# in the templates via {{ site.myvariable }}. -title: Polynote -baseurl: "" # the subpath of your site, e.g. /blog -url: "" # the base hostname & protocol for your site, e.g. http://example.com - -# Build settings -markdown: kramdown -plugins: - - jekyll-feed - - jekyll-relative-links - -relative_links: - enabled: true - collections: false - -# Exclude from processing. -# The following items will not be processed, by default. Create a custom list -# to override the default setting. -# exclude: -# - Gemfile -# - Gemfile.lock -# - node_modules -# - vendor/bundle/ -# - vendor/cache/ -# - vendor/gems/ -# - vendor/ruby/ diff --git a/_includes/footer.html b/_includes/footer.html deleted file mode 100644 index 6a206ec..0000000 --- a/_includes/footer.html +++ /dev/null @@ -1,16 +0,0 @@ - - - \ No newline at end of file diff --git a/_includes/header.html b/_includes/header.html deleted file mode 100644 index c2819d7..0000000 --- a/_includes/header.html +++ /dev/null @@ -1,23 +0,0 @@ - - - - {{ page.title }} - - - - - -

- -

\ No newline at end of file diff --git a/_index.html b/_index.html deleted file mode 100644 index 8a8d1c0..0000000 --- a/_index.html +++ /dev/null @@ -1,150 +0,0 @@ - - - - Polynote | The polyglot Scala notebook - - - -

- -

The polyglot notebook
with first-class Scala support.

- Learn to - - - - your notebooks again. -

Try it!

Features

- Polynote is a different kind of notebook. It supports mixing multiple languages in one notebook, and sharing - data between them seamlessly. It encourages reproducible notebooks with its immutable data model. (Learn more) -

- Here are some of its unique features: -

- -

Editing

Interactive auto-complete helps you find what you're looking for without switching to the documentation.

Highlighting your errors helps you quickly figure out what's gone wrong.

Rich text editing lets you see your formatted text cells as you write them, and edit them like a document.

Easily insert LaTeX equations into your text cells.

Executing

Always know what the kernel is doing with individual task tracking.

The symbol table keeps track of what you've defined, and what's available in the current cell. Inspect any value for detailed information and rich visualizations.

See exactly what code is executing right now, as Polynote highlights the running statement in real-time.

Ordered cell semantics ensures your notebook can be reproducibly executed.

Data and visualization

First-class integration with Apache Spark™ for exploring, analyzing, and visualizing big data.

Browse table-structured data, including collections of data structures and Spark data sets.

See rich representations of your data, automatically derived or supplied by a typeclass.

Easily create visualizations with the built-in plot editor, or define your own with the Vega visualization language.

- - - diff --git a/_layouts/default.html b/_layouts/default.html deleted file mode 100644 index d59ce5c..0000000 --- a/_layouts/default.html +++ /dev/null @@ -1,6 +0,0 @@ -{% include header.html %} -

- {{ content }} -

-{% include footer.html %} diff --git a/_layouts/docs.html b/_layouts/docs.html deleted file mode 100644 index fe423c7..0000000 --- a/_layouts/docs.html +++ /dev/null @@ -1,5 +0,0 @@ -{% include header.html %} -

- {{ content }} -

-{% include footer.html %} diff --git a/_layouts/home.html b/_layouts/home.html deleted file mode 100644 index 3775adb..0000000 --- a/_layouts/home.html +++ /dev/null @@ -1,125 +0,0 @@ - {% include header.html %} -

The polyglot notebook
with first-class Scala support.

- Learn to - - - - your notebooks. -

Try it!

Features

- Here are some of its unique features: -

- -

Editing

Interactive auto-complete helps you find what you're looking for without switching to the documentation.

Highlighting your errors helps you quickly figure out what's gone wrong.

Rich text editing lets you see your formatted text cells as you write them, and edit them like a document.

Easily insert LaTeX equations into your text cells.

Executing

Always know what the kernel is doing with individual task tracking.

The symbol table keeps track of what you've defined, and what's available in the current cell. Inspect any value for detailed information and rich visualizations.

See exactly what code is executing right now, as Polynote highlights the running statement in real-time.

Ordered cell semantics ensures your notebook can be reproducibly executed.

Data and visualization

First-class integration with Apache Spark™ for exploring, analyzing, and visualizing big data.

Browse table-structured data, including collections of data structures and Spark data sets.

See rich representations of your data, automatically derived or supplied by a typeclass.

Easily create visualizations with the built-in plot editor, or define your own with the Vega visualization language.

- {% include footer.html %} diff --git a/_layouts/post.html b/_layouts/post.html deleted file mode 100644 index 86f4fb3..0000000 --- a/_layouts/post.html +++ /dev/null @@ -1,6 +0,0 @@ -{% include header.html %} -

- {{ content }} -

-{% include footer.html %} diff --git a/_posts/2019-10-14-extending-polynote-visualizations.md b/_posts/2019-10-14-extending-polynote-visualizations.md deleted file mode 100644 index 76a1d35..0000000 --- a/_posts/2019-10-14-extending-polynote-visualizations.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -layout: post -excerpt_separator: -title: Extending Polynote with your own visualizations ---- - -We've just published our first snapshot artifacts to Sonatype! These artifacts, `polynote-runtime` and `polynote-spark-runtime`, -are some small dependencies that support integration of other libraries with Polynote's visualization system. - -In this article, we'll look at how you can use these artifacts to build a small library which lets you automatically -render rich visualizations of your own custom data structures in Polynote. - - - -## The goal - -As an example, we're going to add a visualization for [Breeze](https://github.com/scalanlp/breeze) which will display -a `Matrix` using nice mathematical notation. In the end, we want a cell that looks like this: - -```scala -DenseMatrix( - DenseVector(1.0, 2.0, 3.0), - DenseVector(2.0, 3.0, 4.0), - DenseVector(3.0, 4.0, 5.0) -) -``` - -to display its result like this: - -$$ -\begin{bmatrix} -1.0 & 2.0 & 3.0 \\ -2.0 & 3.0 & 4.0 \\ -3.0 & 4.0 & 5.0 -\end{bmatrix} -$$ - -## The code - -First, create a new SBT project: - -``` -$ sbt new scala/scala-seed.g8 -... -A minimal Scala project. - -name [Scala Seed Project]: polynote-viz-example - -Template applied in ./polynote-viz-example - -$ cd polynote-viz-example -``` - -We'll have to edit `build.sbt` in order to set up a few things: - -- Change `scalaVersion` to `2.11.12`, since that's what Polynote currently builds against (hopefully it will be a more recent version soon, but it's limited by what Spark supports). -- Add the Sonatype snapshots repository – where we're currently publishing artifacts: `repositories += Resolver.sonatypeRepo("snapshots")` -- Add Breeze and `polynote-runtime` to the libraryDependencies: `libraryDependencies ++= Seq("org.scalanlp" %% "breeze" % "1.0", "org.polynote" %% "polynote-runtime % "0.2.5-SNAPSHOT")` - -## The `ReprsOf` typeclass - -The machinery behind Polynote's visualizations is driven by a typeclass, [`ReprsOf`](https://github.com/polynote/polynote/blob/master/polynote-runtime/src/main/scala/polynote/runtime/ReprsOf.scala). -It has one method to be implemented: - -```scala -trait ReprsOf[T] extends Serializable { - def apply(value: T): Array[ValueRepr] -} -``` - -Given a value of its target type, it returns an array of `ValueRepr` objects. [`ValueRepr`](https://github.com/polynote/polynote/blob/master/polynote-runtime/src/main/scala/polynote/runtime/ValueRepr.scala) is a sealed trait -which gives several options for how to represent data to the client: - -* `StringRepr` is simply a text value. -* `MIMERepr` is a string value with an associated MIME type string. -* `DataRepr` is a binary-encoded data value with an associated data type. -* `LazyDataRepr` is similar to `DataRepr`, but is lazily evaluated (and each client must ask for it, while `DataRepr`s are eagerly broadcast). -* `StreamingDataRepr` represents a stream of binary data values; each client must request items from the stream. This is used for table-like data. - -For the Breeze matrix, we're going to use `MIMERepr` with a LaTeX MIME type `application/x-latex`. - -## Orphan instances - -In order for Polynote to find `ReprsOf[DenseMatrix[Double]]`, it has to be in implicit scope for that type. So, right away -it seems we have a problem: we can't place this instance in the companion of `ReprsOf` without making `polynote-runtime` -have a dependency on Breeze; nor can we place it in the companion of `DenseMatrix` without making Breeze have a -dependency on `polynote-runtime`. We want to instead define the instance in our new module, which has a dependency -on both. But how can we get the instance into implicit scope? - -The `ReprsOf` typeclass is designed to allow for this. It has a macro in implicit scope, which will search the -dependency classpath for metadata files `META-INF/expanded-scopes/polynote.runtime.ReprsOf`. This metadata file -should contain the fully-qualified name of a trait which extends `ReprsOf`, and that trait's implicit scope will be -effectively added to `ReprsOf`'s. - -Let's create the `src/main/scala` directory, and add a new source file `polynote/breeze/BreezeReprsOf.scala`: - -```scala -package polynote.breeze - -import polynote.runtime.{ReprsOf, ValueRepr, MIMERepr} -import breeze.linalg.DenseMatrix - -trait BreezeReprsOf[T] extends ReprsOf[T] - -object BreezeReprsOf { - implicit val doubleDenseMatrix: BreezeReprsOf[DenseMatrix[Double]] = - new BreezeReprsOf[DenseMatrix[Double]] { - def apply(value: DenseMatrix[Double]): Array[ValueRepr] = Array( - MIMERepr( - "application/x-latex", - - ) - ) - } -} -``` \ No newline at end of file diff --git a/docs/01-installation.md b/docs/01-installation.md deleted file mode 100644 index 6446867..0000000 --- a/docs/01-installation.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Installing Polynote -layout: docs ---- - -# Installing Polynote - -> ## Warning! -> Polynote is a web-based programming notebook tool. Like other notebook tools, a large part of its usefulness relies on -> **arbitrary remote code execution.** -> -> It currently contains **no built-in security or authentication** of its own, and relies entirely on the user deploying -> and configuring it in a secure way. -> -> Polynote should only be deployed on a secure server which has its own security and authentication mechanisms which -> prevent all unauthorized network access. -> -> You are solely responsible for any damage or other loss that occurs as a result of running Polynote. -{:.warning} - -## Download -Polynote consists of a JVM-based server application, which serves a web-based client. To try it locally, find the latest -release on the [releases page](https://github.com/polynote/polynote/releases){:target="_blank"} and download the attached -`polynote-dist.tar.gz` file (you'll find it under `Assets`). Unpack the archive: - -``` -tar -zxvpf polynote-dist.tar.gz -cd polynote -``` - -## Prerequisites -- Polynote is currently only tested on Linux and MacOS, using the Chrome browser as a client. We hope to be testing - other platforms and browsers soon. Feel free to try it on your platform, and be sure to let us know about any issues - you encounter by filing a bug report. - - _Windows:_ Some users have reported that Polynote runs successfully on Windows Subsystem for Linux. Please see these issues for more information: [#555](https://github.com/polynote/polynote/issues/555), [#671](https://github.com/polynote/polynote/issues/671). - - Polynote has been successfully tested on both Java 8 and Java 11. - -- *Spark support*: In order to use Spark with kernel isolation, you'll need to [install Apache Spark™](https://spark.apache.org/downloads.html). - If you'll be using Spark with Polynote, please make sure you read this [note about Spark and Polynote](02-basic-usage.md#using-spark-with-polynote) for more information. - - Polynote will use the `spark-submit` command in order to start isolated kernels, so it needs the `spark-submit` command - to be working properly and available on the `PATH` of the environment you used to launch the server. - - On a Mac with [Homebrew](https://brew.sh), you can install Spark locally with `brew install apache-spark`. - - On Linux, untar Spark wherever you like and configure your environment to have `SPARK_HOME` pointing to the Spark location. - Then add `$SPARK_HOME/bin` to your `PATH`. You'll also need a JDK installed and `JAVA_HOME` properly configured. - Here is an example setup for Polynote on Debian. Note that distributions differ, so you may need to modify these - instructions for your local setup. - - Start with an `apt-get update` - - Install Java (this installs Java 11 for me on Debian buster): `apt-get install default-jdk` - - download polynote, `tar -zxvpf polynote-dist.tar.gz` - - download spark, `tar -zxvf spark-2.4.4-bin-hadoop2.7.tgz` - - Set `JAVA_HOME`: `export JAVA_HOME=/usr/lib/jvm/default-java/` - - Set `SPARK_HOME`: ``export SPARK_HOME=`pwd`/spark-2.4.4-bin-hadoop2.7/`` - - Set `PATH`: `export PATH="$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin"` - - Check if Spark is correctly set up by running `spark-submit`, you should see usage information. - - Do the Python setup as described below. You might need to install `build-essential` if jep installation fails. - - [Run Polynote!](#Run) - -- *Python support*: In order to execute Python code, you'll need to have Python 3 and pip3 installed. Refer to - [Python's installation instructions](https://wiki.python.org/moin/BeginnersGuide/Download){:target="_blank"} for - instructions on installing these packages. - - You'll also need to install Polynote's Python dependencies: - - ``` - pip3 install -r ./requirements.txt - ``` - - -## Configure -To change any of the default configuration, you'll need to copy the included [`config-template.yml`](https://github.com/polynote/polynote/blob/master/config-template.yml) -file to `config.yml`, and uncomment the sections you'd like to change. Check out the template itself for more information. - -## Run -To start the server, run the included python script: - -``` -./polynote.py -``` - -## Next -Once the server has started, navigate your browser to `http://localhost:8192` (if using the default network configuration). -Next, read about [basic usage](02-basic-usage.md) diff --git a/docs/02-basic-usage.md b/docs/02-basic-usage.md deleted file mode 100644 index 9a44a02..0000000 --- a/docs/02-basic-usage.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: Basic usage -layout: docs ---- - -# Basic usage - -After starting Polynote and opening the application in your browser, you'll see the main interface: - -![images/fresh-home-screen.png](images/fresh-home-screen.png) - -On the left is the list of notebook files. If you have just installed Polynote, the only notebooks you will have access to are our examples. Feel free to browse these notebooks for your future reference. You can also import -existing Jupyter notebooks into Polynote – simply drag-and-drop the notebook file from your computer onto the Notebooks -panel. - -For now, though, let's just create a new notebook. Click the "New notebook" icon as the home screen instructs, and enter a name -(such as "First Notebook"). Click "OK". The new notebook appears in the Notebooks pane, and is opened in a Polynote -tab. The new notebook contains a single text cell. Click on that and you'll see the text editing toolbar appear: - -![images/text-editing.png](images/text-editing.png) - -This contextual toolbar appears when it's useful – when editing a text cell. A notebook is made up of text cells and -code cells; text cells are useful for writing rich text explanations, and code cells are what actually get run. Play -around with the text formatting tools for a bit, and then we'll create a new cell by pressing `Shift`{:.key}-`Enter`{:.key}. -The new cell is a code cell, with its language (by default) set to Scala: - -![images/code-cell.png](images/code-cell.png) - -You can change the language with the dropdown – there's one right above the cell and one on the main toolbar. If you -wanted another text cell instead, you can change it to Text as well (but only in the dropdown on the main toolbar) and -it will turn into a text cell. - -Let's enter some Scala code in the code cell. Once you start typing, you'll see some activity on the right as the Kernel -is started. The Kernel is what handles the code cells. Once it starts, you'll start seeing interactive completions -as you type code. Run the code cell by pressing `Shift`{:.key}-`Enter`{:.key}. The cell runs, and output is displayed: - -![images/results.png](images/results.png) - -We can see that a few things happened after running the code: - -- The console output from the `println` statement appeared in an area below the cell. -- The variable `wizzle` defined in the cell appeared in the symbol table on the right. -- The final expression of the cell was bound to a variable called `Out`, and some information about it appeared below - the console output. - -Because the expression `List.fill(50)(Point(nextGaussian * 2 + 5, nextGaussian * 4))` results in "table-like" data – -a collection of structured values – there is a special representation of it displayed. We see its "schema" (the fields -of the structured "row"), along with its type and some icons. These icons bring up the inspection window, to either -browse or plot the data (we'll go into more detail about this, but try playing with it if you like). In addition to -inspecting the "result" of the cell, any other value that's been bound to a variable can be inspected by clicking it in -the symbol table. - -## The symbol table and input scope - -If you create and run some more code cells, you'll notice that the symbol table changes depending on which cell is -currently focused. It shows you any values that resulted from the current cell above a thick black line, and any values -that are available to the cell (from previous cells) below the thick black line. - -Polynote is different from other notebook tools in that a cell's place in the notebook is important. Every cell has an -"input state" defined by the cells that have run above it. This means that if we create a code cell that defines a value: - -```scala -val foo = 20 -``` - -Then another cell below it can use the value `foo`: - -```scala -val bar = foo * 2 -``` - -But, if we then go back and edit the first cell like this: - -```scala -val foo = 20 -val fail = bar * 2 -``` - -It won't work! The value `bar` isn't in the input scope of the first cell, because it was defined in the second cell. -This is very different from how other notebook tools operate; they just have one global state that every cell mutates. -In Polynote, the input state of a cell is defined completely by which cells are above it. This is a powerful way to -enforce *reproducibility* in a notebook; it is far more likely that you'll be able to re-run the notebook from top to -bottom if later cells can't affect earlier cells. - -## Using Spark with Polynote - -Polynote has deep integration with [Apache Spark](https://spark.apache.org). However, it can also run without Spark support enabled. -In fact, since launching a Spark session can be expensive, Polynote runs your notebooks without Spark support by default. - -Polynote attempts to be smart about whether to launch a notebook with Spark: it'll only launch a Spark session if you -have any Spark configuration parameters in the notebook config. So the fastest way to get started with Spark is to add -something like `spark.master: local[*]` to your notebook configuration, like so: - -![spark-master-config](images/spark-master-config.png) - -Next, read about [mixing programming languages](03-mixing-programming-languages.md). diff --git a/docs/03-mixing-programming-languages.md b/docs/03-mixing-programming-languages.md deleted file mode 100644 index 836b010..0000000 --- a/docs/03-mixing-programming-languages.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: Mixing programming languages -layout: docs ---- - -## Mixing programming languages - -One of Polynote's most interesting features is its support for polyglot notebooks, where cells within the same notebook -can be written in different languages. For example, a notebook could do some data manipulation with Scala and Spark, -and then plot that data in Python with Matplotlib. - -We find that this lets us do some pretty neat things, but it does have some limitations and caveats, and plenty of edge-cases -left to be worked out. - -### Execution state in Polynote - -We hinted [previously](02-basic-usage.md#The-symbol-table-and-input-scope) that the way Polynote handles cell execution -is a little special. - -As a reminder, the kernel keeps track of all the symbols defined by a cell execution. These symbols are part -of the cell's state, which is made available to downstream cells (those below the cell in question) when they in turn -are executed. - -Polynote stores these symbols, alongside their types and other information, in a Scala-based format. Symbols defined by -and provided to other languages are wrapped (and unwrapped) appropriately. - -### Sharing between Python and Scala - -For now, Python is the major non-JVM-based language you'll be using with Polynote. Polynote uses -[jep](https://github.com/ninia/jep) to do most of the heavy-lifting when it comes to Python interop. If you're going to -be moving back and forth between Python and Scala a lot, we highly recommend at least reading about -[how Jep works](https://github.com/ninia/jep/wiki/How-Jep-Works). - -Our goals right now are to support a few, key use-cases with a focus on sharing from Scala to Python, -such as plotting data generated in Scala with `matplotlib`, or using Scala-generated data with `tensorflow` and -`scikit-learn`. We've found that the interop between Python and Scala can be very powerful even if its limited to these -simple cases. - -Here are a few important points to keep in mind when sharing between Python and Scala: - -* Jep handles the conversion from Scala -> Python. - * It converts primitives and strings into brand-new Python primitives and strings. - * An object of any other type is wrapped as a `PyJObject`, which is an interface allowing Python to directly access - that objects attributes. Note that in this case, nothing is copied - `PyJObject` holds a reference to the underlying - JVM object. - * Note that Jep is based on Java, not Scala. This means that when it wraps a Scala object as a `PyJObject`, you won't - get Scala sugar - things like multiple parameter lists, implicits, etc. - when you work with it in Python. - This can limit your ability to use a lot of super-scala-stuff with Python. -* Jep handles conversion from Python -> Scala and Polynote adds a little bit of sugar on top. - * Similar to the other way round, Jep automatically converts primitives and strings into brand-new JVM primitives and strings. - * Additionally, Jep supports some other conversions such as Python `dict` to `java.util.HashMap` - * Polynote will retrieve an object of any other type as a `PyObject`. Similar to `PyJObject`, a `PyObject` wraps a pointer - to a Python object. Polynote has some support for handling certain types of Python objects, typically for visualization - purposes. - -Note that these implementation details may change and while we'll work hard to update this information we can't guarantee -that it won't get out-of-date. Of course, feel free to [drop us a line](https://gitter.im/polynote/polynote) if you -think that's the case! - -### Cookbook - -We have a bunch of example notebooks over in the [examples folder](https://github.com/polynote/polynote/tree/master/docs/examples), -showcasing various useful tricks and things to keep in mind while working with Polynote. - diff --git a/CNAME b/docs/CNAME similarity index 100% rename from CNAME rename to docs/CNAME diff --git a/about.md b/docs/about.md similarity index 94% rename from about.md rename to docs/about.md index 1b699e0..da98c3c 100644 --- a/about.md +++ b/docs/about.md @@ -1,6 +1,6 @@ --- layout: default -title: About Polynote +title: About permalink: /about/ --- diff --git a/contribute.md b/docs/contribute.md similarity index 98% rename from contribute.md rename to docs/contribute.md index e190849..a0ba90d 100644 --- a/contribute.md +++ b/docs/contribute.md @@ -1,6 +1,6 @@ --- layout: default -title: Contribute to Polynote +title: Contribute permalink: /contribute/ --- diff --git a/docs/docs/about-menu.md b/docs/docs/about-menu.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/advanced-polyglot.md b/docs/docs/advanced-polyglot.md new file mode 100644 index 0000000..5724883 --- /dev/null +++ b/docs/docs/advanced-polyglot.md @@ -0,0 +1,35 @@ + +### Sharing between Python and Scala + +For now, Python is the major non-JVM-based language you'll be using with Polynote. Polynote uses +[jep](https://github.com/ninia/jep) to do most of the heavy-lifting when it comes to Python interop. If you're going to +be moving back and forth between Python and Scala a lot, we highly recommend reading about +[how Jep works](https://github.com/ninia/jep/wiki/How-Jep-Works). + +Our goals right now are to support a few, key use-cases with a focus on sharing from Scala to Python, +such as plotting data generated in Scala with `matplotlib`, or using Scala-generated data with `tensorflow` and +`scikit-learn`. We've found that the interop between Python and Scala can be very powerful even if it is limited to these +simple cases. + +TODO: Break this out into "Advanced" section? + +Here are a few important points to keep in mind when sharing between Python and Scala: + +* Jep handles the conversion from Scala -> Python. + * It converts primitives and strings into brand-new Python primitives and strings. + * An object of any other type is wrapped as a `PyJObject`, which is an interface allowing Python to directly access + that objects attributes. Note that in this case, nothing is copied - `PyJObject` holds a reference to the underlying + JVM object. + * Note that Jep is based on Java, not Scala. This means that when it wraps a Scala object as a `PyJObject`, you won't + get Scala sugar - things like multiple parameter lists, implicits, etc. - when you work with it in Python. + This can limit your ability to use a lot of super-scala-stuff with Python. +* Jep handles conversion from Python -> Scala and Polynote adds a little bit of sugar on top. + * Similar to the other way round, Jep automatically converts primitives and strings into brand-new JVM primitives and strings. + * Additionally, Jep supports some other conversions such as Python `dict` to `java.util.HashMap` + * Polynote will retrieve an object of any other type as a `PyObject`. Similar to `PyJObject`, a `PyObject` wraps a pointer + to a Python object. Polynote has some support for handling certain types of Python objects, typically for visualization + purposes. + +Note that these implementation details may change and while we'll work hard to update this information we can't guarantee +that it won't get out-of-date. Of course, feel free to [drop us a line](https://gitter.im/polynote/polynote) if you +think that's the case! \ No newline at end of file diff --git a/docs/docs/basic-usage.md b/docs/docs/basic-usage.md new file mode 100644 index 0000000..4b71f1c --- /dev/null +++ b/docs/docs/basic-usage.md @@ -0,0 +1,160 @@ +# Basic usage + +!!! info + This page aims to get you up and running with your first Polynote notebook. For a more in-depth walk through the + Polynote user interface, check out the [Tour](tour.md). + +After starting Polynote and opening the application in your browser at http://localhost:8192/ +you'll see the home page. + +![Fresh Home Screen](images/fresh-home-screen.png) + +On the left is the list of notebook files. If you have just installed Polynote, the only notebooks you will have access +to are our examples. Feel free to browse these notebooks for your future reference. + +## Creating a new notebook + +For now, though, we'll just jump right into creating a new notebook. + +![Create notebook button](images/create-notebook-btn.png){ align=left style="height:8em" } + +![Create notebook dialog](images/create-notebook-dialog.png){ align=right style="height:8em" } + +Click the "New notebook" button (left), and a dialog will come up (right). Enter a name for your notebook +and then click "Create". + +![New notebook configuration](images/new-nb-detail.png) + +The new notebook appears in the Notebooks Pane ①, and is opened in a Polynote tab +② with the Configuration Pane open ③. + +We'll ignore the configuration for now, so click anywhere on the `▼ Configuration & dependencies` heading +④ to close it. + +You'll see that the notebook contains a single text cell. Click on that cell and you'll see the text editing toolbar +appear (shown boxed in the image below). + +![images/text-editing.png](images/text-editing.png) + +This contextual toolbar appears when it's useful, based on the currently active cell. You'll see it change as you +interact with cells of various types. + +!!!tip "" + Check out the [tour](toolbar.md) for a more comprehensive breakdown of the toolbar and the different cell types. + +## Creating a new cell + +A notebook document is made up of a collection of cells. Currently, Polynote supports three cell types: text cells, +code cells, and visualization (viz) cells. + +[Text cells](text-cells.md) are useful for writing rich text explanations, [code cells](code-cells.md) contain +the code that actually gets run in the notebook, and [viz cells](visualization-and-viz-cells.md) aid in inspecting and visualizing data. + +Now that we've seen a text cell, let's create a new code cell. If you hover the mouse above and below the text cell, +you'll see a divider appear with a small `+` symbol. Clicking on that divider will create a new cell in its place. + +Go ahead and click on the divider below the text cell to create a new cell there. + +![Creating a new cell](images/polynote-new-cell-click.gif) + +This new cell is a code cell, with its language (by default) set to Scala. You can change the language with the dropdown +– there's one right above the cell and one on the main toolbar. + +If you wanted another text cell instead, you can change it to Text as well (but only in the dropdown on the main toolbar) +and it will turn into a text cell. + +![Changing cell language](images/polynote-change-cell-lang.gif) + +Let's enter some Scala code in the code cell. The code we're writing is just a little snippet for demonstration purposes. +Here it is in case you're following along: + + +???example "Demonstation Code" + ```scala + import scala.util.Random.nextGaussian + + println("this is some console output!") + + val x = 50 + + case class Point(x: Double, y: Double) + List.fill(x)(Point(nextGaussian * 2 + 5, nextGaussian * 4)) + ``` + +Once you start typing, you'll see some activity on the right as the Kernel is started. The Kernel is what handles the +computing the notebook code, including providing code completions and suggestions. Once it starts, you'll start seeing +interactive completions as you type. + +Run the code cell by pressing ++shift+enter++. The cell runs, and output is displayed. + +Here's what you should see: + +![Editing and running a code cell](images/polynote-code-cell.gif) + +Let's take a closer look at the results of running the cell. + +![The results of a code cell](images/results.png) + +We can see that a few things happened after running the code: + +- The console output from the `println` statement appeared in an area ① below the cell. +- The variable `x` defined in the cell appeared in the Symbol Table ② on the right. +- The final expression of the cell was bound to a variable called `Out` ③ and some information + about it appeared below the console output ④. + +Because the expression `:::scala List.fill(50)(Point(nextGaussian * 2 + 5, nextGaussian * 4))` results in "table-like" +data – a collection of structured values – a special representation of it is displayed. In this Cell Result view, we +can see the result's schema (the fields of the structured row) along with its type and some icons. + +Clicking these icons creates a new viz cell below this one, allowing you to inspect the cell's result in more detail. + +In addition to inspecting the "result" of the cell, any other value that's been bound to a variable can be inspected by +clicking it in the Symbol Table. We'll now take a closer look at the Symbol Table, and how it changes as you interact +with the notebook. + +## The Symbol Table and input scope + +If you create and run some more code cells, you'll notice that the Symbol Table changes depending on which cell is +currently focused. It shows you any values that resulted from the current cell above a thick black line, and any values +that are available to the cell (from previous cells) below the thick black line. + +![The Symbol Table shows you the current input scope](images/symbol-table-input-scope.gif) + +In the image above, you can see how the Symbol Table changes based on which cell is selected. This allows you to keep +track of which variables are available, which is useful because Polynote keeps track of the cell input scope. + +Polynote is different from other notebook tools in that a cell's place in the notebook is important. Every cell has an +"input state" defined by the cells that have run above it. This means that if we create a code cell that defines a value: + +```scala +val foo = 20 +``` + +Then another cell below it can use the value `foo`: + +```scala +val bar = foo * 2 +``` + +But, if we then go back and edit the first cell like this: + +```scala +val foo = 20 +val fail = bar * 2 +``` + +It won't work! + +The value `bar` isn't in the input scope of the first cell, because it was defined in the second cell. +This is very different from how other notebook tools operate; they just have one global state that every cell mutates. + +In Polynote, the input state of a cell is defined completely by which cells are above it. This is a powerful way to +enforce **reproducibility** in a notebook; it is far more likely that you'll be able to re-run the notebook from top to +bottom if later cells can't affect earlier cells. + +Here's an example of the above scenario in practice (reordering the cells rather than editing their content): + +![Moving cell with variable reference out of scope](images/move-input-scope.gif) + + +Next, read about [mixing programming languages](mixing-programming-languages.md). diff --git a/docs/docs/code-cells.md b/docs/docs/code-cells.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md new file mode 100644 index 0000000..94d7560 --- /dev/null +++ b/docs/docs/configuration.md @@ -0,0 +1,4 @@ +To change any of the default configuration, you'll need to copy the included [`config-template.yml`](https://github.com/polynote/polynote/blob/master/config-template.yml) +file to `config.yml`, and uncomment the sections you'd like to change. Check out the template itself for more information. + +### Storage \ No newline at end of file diff --git a/docs/docs/development.md b/docs/docs/development.md new file mode 100644 index 0000000..999c9e1 --- /dev/null +++ b/docs/docs/development.md @@ -0,0 +1 @@ +`sbt dist` \ No newline at end of file diff --git a/docs/docs/docker.md b/docs/docs/docker.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/examples/Accessing a Python object in Scala.ipynb b/docs/docs/examples/Accessing a Python object in Scala.ipynb similarity index 100% rename from docs/examples/Accessing a Python object in Scala.ipynb rename to docs/docs/examples/Accessing a Python object in Scala.ipynb diff --git a/docs/examples/Accessing a Scala object in Python.ipynb b/docs/docs/examples/Accessing a Scala object in Python.ipynb similarity index 100% rename from docs/examples/Accessing a Scala object in Python.ipynb rename to docs/docs/examples/Accessing a Scala object in Python.ipynb diff --git a/docs/examples/Case class access in Python.ipynb b/docs/docs/examples/Case class access in Python.ipynb similarity index 100% rename from docs/examples/Case class access in Python.ipynb rename to docs/docs/examples/Case class access in Python.ipynb diff --git a/docs/examples/Hello World.ipynb b/docs/docs/examples/Hello World.ipynb similarity index 100% rename from docs/examples/Hello World.ipynb rename to docs/docs/examples/Hello World.ipynb diff --git a/docs/examples/Plotting Scala data with Matplotlib.ipynb b/docs/docs/examples/Plotting Scala data with Matplotlib.ipynb similarity index 100% rename from docs/examples/Plotting Scala data with Matplotlib.ipynb rename to docs/docs/examples/Plotting Scala data with Matplotlib.ipynb diff --git a/docs/examples/Scala Spark to Pandas.ipynb b/docs/docs/examples/Scala Spark to Pandas.ipynb similarity index 100% rename from docs/examples/Scala Spark to Pandas.ipynb rename to docs/docs/examples/Scala Spark to Pandas.ipynb diff --git a/docs/docs/faq.md b/docs/docs/faq.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/images/clear-output.png b/docs/docs/images/clear-output.png new file mode 100644 index 0000000..18ef1b6 Binary files /dev/null and b/docs/docs/images/clear-output.png differ diff --git a/docs/docs/images/code-cell.png b/docs/docs/images/code-cell.png new file mode 100644 index 0000000..876df97 Binary files /dev/null and b/docs/docs/images/code-cell.png differ diff --git a/docs/docs/images/collapse-pane.gif b/docs/docs/images/collapse-pane.gif new file mode 100644 index 0000000..a5ef656 Binary files /dev/null and b/docs/docs/images/collapse-pane.gif differ diff --git a/docs/docs/images/context-menu-rename.gif b/docs/docs/images/context-menu-rename.gif new file mode 100644 index 0000000..7804e50 Binary files /dev/null and b/docs/docs/images/context-menu-rename.gif differ diff --git a/docs/docs/images/create-notebook-btn.png b/docs/docs/images/create-notebook-btn.png new file mode 100644 index 0000000..9bd3f4d Binary files /dev/null and b/docs/docs/images/create-notebook-btn.png differ diff --git a/docs/docs/images/create-notebook-dialog.png b/docs/docs/images/create-notebook-dialog.png new file mode 100644 index 0000000..1f04b3a Binary files /dev/null and b/docs/docs/images/create-notebook-dialog.png differ diff --git a/docs/docs/images/download-current-notebook.png b/docs/docs/images/download-current-notebook.png new file mode 100644 index 0000000..a5535c3 Binary files /dev/null and b/docs/docs/images/download-current-notebook.png differ diff --git a/docs/docs/images/drag-drop-notebook.gif b/docs/docs/images/drag-drop-notebook.gif new file mode 100644 index 0000000..67ce91c Binary files /dev/null and b/docs/docs/images/drag-drop-notebook.gif differ diff --git a/docs/docs/images/fresh-home-screen.png b/docs/docs/images/fresh-home-screen.png new file mode 100644 index 0000000..8af2b65 Binary files /dev/null and b/docs/docs/images/fresh-home-screen.png differ diff --git a/docs/docs/images/green-plus.png b/docs/docs/images/green-plus.png new file mode 100644 index 0000000..f0ec4f0 Binary files /dev/null and b/docs/docs/images/green-plus.png differ diff --git a/docs/docs/images/import scala.png b/docs/docs/images/import scala.png new file mode 100644 index 0000000..8e20ca2 Binary files /dev/null and b/docs/docs/images/import scala.png differ diff --git a/docs/docs/images/inspecting-idiomaticData.gif b/docs/docs/images/inspecting-idiomaticData.gif new file mode 100644 index 0000000..ccaa2a4 Binary files /dev/null and b/docs/docs/images/inspecting-idiomaticData.gif differ diff --git a/docs/docs/images/move-input-scope.gif b/docs/docs/images/move-input-scope.gif new file mode 100644 index 0000000..70a9dec Binary files /dev/null and b/docs/docs/images/move-input-scope.gif differ diff --git a/docs/docs/images/new-nb-configuration.png b/docs/docs/images/new-nb-configuration.png new file mode 100644 index 0000000..f737885 Binary files /dev/null and b/docs/docs/images/new-nb-configuration.png differ diff --git a/docs/docs/images/new-nb-detail.png b/docs/docs/images/new-nb-detail.png new file mode 100644 index 0000000..abb46fa Binary files /dev/null and b/docs/docs/images/new-nb-detail.png differ diff --git a/docs/docs/images/notebooks-pane.png b/docs/docs/images/notebooks-pane.png new file mode 100644 index 0000000..df10c06 Binary files /dev/null and b/docs/docs/images/notebooks-pane.png differ diff --git a/docs/docs/images/plotting-scala-matplotlib-idiomatic.png b/docs/docs/images/plotting-scala-matplotlib-idiomatic.png new file mode 100644 index 0000000..1f58641 Binary files /dev/null and b/docs/docs/images/plotting-scala-matplotlib-idiomatic.png differ diff --git a/docs/docs/images/plotting-scala-matplotlib.png b/docs/docs/images/plotting-scala-matplotlib.png new file mode 100644 index 0000000..a162802 Binary files /dev/null and b/docs/docs/images/plotting-scala-matplotlib.png differ diff --git a/docs/docs/images/polynote-change-cell-lang.gif b/docs/docs/images/polynote-change-cell-lang.gif new file mode 100644 index 0000000..2c74e9a Binary files /dev/null and b/docs/docs/images/polynote-change-cell-lang.gif differ diff --git a/docs/docs/images/polynote-code-cell.gif b/docs/docs/images/polynote-code-cell.gif new file mode 100644 index 0000000..2ec221c Binary files /dev/null and b/docs/docs/images/polynote-code-cell.gif differ diff --git a/docs/docs/images/polynote-new-cell-click.gif b/docs/docs/images/polynote-new-cell-click.gif new file mode 100644 index 0000000..82d73bb Binary files /dev/null and b/docs/docs/images/polynote-new-cell-click.gif differ diff --git a/docs/docs/images/polynote-ui-labelled.png b/docs/docs/images/polynote-ui-labelled.png new file mode 100644 index 0000000..3df4909 Binary files /dev/null and b/docs/docs/images/polynote-ui-labelled.png differ diff --git a/docs/docs/images/python-scala-seq.png b/docs/docs/images/python-scala-seq.png new file mode 100644 index 0000000..2956a44 Binary files /dev/null and b/docs/docs/images/python-scala-seq.png differ diff --git a/docs/docs/images/resize-pane.gif b/docs/docs/images/resize-pane.gif new file mode 100644 index 0000000..974b853 Binary files /dev/null and b/docs/docs/images/resize-pane.gif differ diff --git a/docs/docs/images/results.png b/docs/docs/images/results.png new file mode 100644 index 0000000..fbed4c5 Binary files /dev/null and b/docs/docs/images/results.png differ diff --git a/docs/docs/images/run-all-cells.png b/docs/docs/images/run-all-cells.png new file mode 100644 index 0000000..2fe8ea3 Binary files /dev/null and b/docs/docs/images/run-all-cells.png differ diff --git a/docs/docs/images/simple-py-interop.png b/docs/docs/images/simple-py-interop.png new file mode 100644 index 0000000..1f0d3a2 Binary files /dev/null and b/docs/docs/images/simple-py-interop.png differ diff --git a/docs/docs/images/spark-master-config.png b/docs/docs/images/spark-master-config.png new file mode 100644 index 0000000..3546b36 Binary files /dev/null and b/docs/docs/images/spark-master-config.png differ diff --git a/docs/docs/images/symbol-table-input-scope.gif b/docs/docs/images/symbol-table-input-scope.gif new file mode 100644 index 0000000..2100167 Binary files /dev/null and b/docs/docs/images/symbol-table-input-scope.gif differ diff --git a/docs/docs/images/text-editing.png b/docs/docs/images/text-editing.png new file mode 100644 index 0000000..2cd7f03 Binary files /dev/null and b/docs/docs/images/text-editing.png differ diff --git a/docs/docs/images/toolbar-about.png b/docs/docs/images/toolbar-about.png new file mode 100644 index 0000000..a058908 Binary files /dev/null and b/docs/docs/images/toolbar-about.png differ diff --git a/docs/docs/images/toolbar-cell-delete.png b/docs/docs/images/toolbar-cell-delete.png new file mode 100644 index 0000000..aeed29e Binary files /dev/null and b/docs/docs/images/toolbar-cell-delete.png differ diff --git a/docs/docs/images/toolbar-cell-insert-above.png b/docs/docs/images/toolbar-cell-insert-above.png new file mode 100644 index 0000000..930a81c Binary files /dev/null and b/docs/docs/images/toolbar-cell-insert-above.png differ diff --git a/docs/docs/images/toolbar-cell-insert-below.png b/docs/docs/images/toolbar-cell-insert-below.png new file mode 100644 index 0000000..749df8d Binary files /dev/null and b/docs/docs/images/toolbar-cell-insert-below.png differ diff --git a/docs/docs/images/toolbar-cell-section.png b/docs/docs/images/toolbar-cell-section.png new file mode 100644 index 0000000..3c6ee45 Binary files /dev/null and b/docs/docs/images/toolbar-cell-section.png differ diff --git a/docs/docs/images/toolbar-code-cell.png b/docs/docs/images/toolbar-code-cell.png new file mode 100644 index 0000000..2e270e5 Binary files /dev/null and b/docs/docs/images/toolbar-code-cell.png differ diff --git a/docs/docs/images/toolbar-code-run-until-selected.png b/docs/docs/images/toolbar-code-run-until-selected.png new file mode 100644 index 0000000..8c009c4 Binary files /dev/null and b/docs/docs/images/toolbar-code-run-until-selected.png differ diff --git a/docs/docs/images/toolbar-code-run.png b/docs/docs/images/toolbar-code-run.png new file mode 100644 index 0000000..23d52b5 Binary files /dev/null and b/docs/docs/images/toolbar-code-run.png differ diff --git a/docs/docs/images/toolbar-code-stop.png b/docs/docs/images/toolbar-code-stop.png new file mode 100644 index 0000000..0d06549 Binary files /dev/null and b/docs/docs/images/toolbar-code-stop.png differ diff --git a/docs/docs/images/toolbar-notebook-section.png b/docs/docs/images/toolbar-notebook-section.png new file mode 100644 index 0000000..ac8dcd1 Binary files /dev/null and b/docs/docs/images/toolbar-notebook-section.png differ diff --git a/docs/docs/images/toolbar-sections.png b/docs/docs/images/toolbar-sections.png new file mode 100644 index 0000000..440c7c1 Binary files /dev/null and b/docs/docs/images/toolbar-sections.png differ diff --git a/docs/docs/images/toolbar-text-cell.png b/docs/docs/images/toolbar-text-cell.png new file mode 100644 index 0000000..bac5649 Binary files /dev/null and b/docs/docs/images/toolbar-text-cell.png differ diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 0000000..06b62b6 --- /dev/null +++ b/docs/docs/index.md @@ -0,0 +1,11 @@ +# Getting Started + +Welcome to Polynote! Polynote is a different kind of notebook. + +It supports mixing multiple languages in one notebook, and sharing data between them seamlessly. +It encourages reproducible notebooks with its immutable data model. + +Get started by [installing Polynote](installation.md), or by taking a [tour](tour.md) of the UI! + +When you're ready, check out the [User Guide](basic-usage.md) for more information. Or if you need to customize your +installation, head over to the [configuration](configuration.md) section. \ No newline at end of file diff --git a/docs/docs/installation.md b/docs/docs/installation.md new file mode 100644 index 0000000..ed23072 --- /dev/null +++ b/docs/docs/installation.md @@ -0,0 +1,205 @@ +# Installing Polynote + +!!!danger + Polynote allows **arbitrary remote code execution**, which is necessary for a notebook tool to function. + While we'll try to improve safety by adding security measures, it will never be completely safe to + run Polynote on your personal computer. For example: + + - It's possible that other websites you visit could use Polynote as an attack vector. Browsing the web + while running Polynote is unsafe. + - It's possible that remote attackers could use Polynote as an attack vector. Running Polynote on a + computer that's accessible from the internet is unsafe. + - Even running Polynote inside a container doesn't guarantee safety, as there will always be + privilege escalation and container escape vulnerabilities which an attacker could leverage. + + Please be diligent about checking for new releases, as they could contain fixes for critical security + flaws. + + Please be mindful of the security issues that Polynote causes; consult your company's security team + before running Polynote. You are solely responsible for any breach, loss, or damage caused by running + this software insecurely. + + +Polynote is currently only tested on Linux and MacOS, using the Chrome browser as a client. + +While we'd love it if Polynote would work perfectly on other platforms and browsers, our small team doesn't have the +bandwidth to test every possible combination. + +Feel free to try it on your platform, and be sure to let us know about any issues you encounter by filing a bug report. + +!!! help "What about Firefox?" + While we don't test with Firefox, we are very interested in making sure that Firefox is a viable alternative to + Chrome when using Polynote. + + Right now, we're in need of Firefox users who can provide us with bug reports (and, ideally, bug _fixes_ too!), so + please do let us know if you run into issues using Firefox! + +!!! help "What about Windows?" + Some users have reported that Polynote runs successfully on Windows Subsystem for Linux. We have no plans to support + Windows outside of the WSL. + + Please see these issues for more information: [#555](https://github.com/polynote/polynote/issues/555), [#671](https://github.com/polynote/polynote/issues/671). + +!!! tip "Using Polynote with Docker" + Instead of installing Polynote on your computer, you can run it within a Docker container. This is a good way to get + up and running quickly, though properly configuring the networking settings so a Docker container running on your + laptop can access a remote Spark cluster can be a bit tricky. + + For more information about using Docker to run Polynote, see the [Polynote Docker docs](docker.md). + +## Prerequisites + +Polynote has been successfully tested on both Java 8 and Java 11. + +### Spark support + +In order to use [Apache Spark™](https://spark.apache.org/) with Polynote, you'll need to have it set up in your +environment. + +!!! tip "Using Spark with Polynote" + If you'll be using Spark with Polynote, please make sure you read this [note about Spark and Polynote](basic-usage.md#using-spark-with-polynote) for more information. + + Currently, Polynote supports both **Spark 2.1** (with Scala 2.11) and **2.4** (with Scala 2.11 and 2.12). + + _Some users have had success running Spark 3.0 with Scala 2.12. Please see [this issue](https://github.com/polynote/polynote/issues/926) for more information_ + +Polynote will use the `spark-submit` command in order to start Spark kernels. + +The `spark-submit` command must be working properly and available on the `PATH` of the environment you used to launch the +server in order for Polynote to work properly with Spark. + +??? info "Installing Spark in your environment" + + These instructions are provided as a convenience, but please note that versions and environments differ. Please + refer to the official documentation or other online resources if you run into trouble setting up Spark. + + As mentioned above, Polynote just needs `spark-submit` to be working correctly and available on the `PATH` in the + environment in which it is running. It doesn't matter how you install Spark as long as those two things are true. + + === "Standard Spark installation" + The "standard" way to install Spark is just to extract the distribution somewhere and then set up some environment variables. + This method works for on both Mac and Linux. + + You may need to modify these instructions based on your local setup. + + === "Spark on Mac" + Here is an example setup for Polynote on Mac. Also, check out the "Installation with Homebrew" tab above for an + alternative installation method. + + Install Java (this installs Java 11) + + brew cask install adoptopenjdk11 + + [Download spark](https://spark.apache.org/downloads.html) and then untar it wherever you like + + tar -zxvf spark-2.4.7-bin-hadoop2.7.tgz + + Set up the `SPARK_HOME` and `PATH` environment variables. You probably want to put these instructions in your + shell configuration somewhere, i.e., `~/.bashrc` or equivalent. + + Set `SPARK_HOME` + + export SPARK_HOME=`pwd`/spark-2.4.7-bin-hadoop2.7/ + + Set `PATH` + + export PATH="$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin" + + Check if Spark is correctly set up by running `spark-submit`, you should see usage information. + + === "Spark on Linux" + Here is an example setup for Polynote on Debian. If you are not using Debian or Ubuntu, you may need to + modify these instructions for your distribution. + + As always, start with updating your package listing. + + apt-get update + + Install Java (this installs Java 11 for me on Debian buster) + + apt-get install default-jdk + + [Download spark](https://spark.apache.org/downloads.html) and then untar it wherever you like + + tar -zxvf spark-2.4.4-bin-hadoop2.7.tgz + + Set up the `JAVA_HOME`, `SPARK_HOME` and `PATH` environment variables. You probably want to put these instructions + in your shell configuration somewhere, i.e., `~/.bashrc` or equivalent. + + Set `JAVA_HOME` + + export JAVA_HOME=/usr/lib/jvm/default-java/ + + Set `SPARK_HOME` + + export SPARK_HOME=`pwd`/spark-2.4.7-bin-hadoop2.7/ + + Set `PATH` + + export PATH="$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin" + + Check if Spark is correctly set up by running `spark-submit`, you should see usage information. + + === "Installing Spark on Mac with Homebrew" + The [Homebrew](https://brew.sh) package manager for Mac has a [formula for Spark](https://formulae.brew.sh/formula/apache-spark), + but unfortunately it only installs the latest version which may not be compatible with Polynote. + + Installing an older version of Spark using Homebrew can be a bit of a pain. There is a third party repository that + has older Spark versions available for easy download. + + !!! warning + It is _highly_ recommended that you read through the homebrew formula before installing it - especially when using + a third-party repository. The [entire thing](https://github.com/eddies/homebrew-spark-tap/blob/master/Formula/apache-spark%402.4.6.rb) + is less than 30 lines of Ruby code and quite readable. + + You can view the available formulae here: https://github.com/eddies/homebrew-spark-tap/tree/master/Formula + + Here's an example for Spark 2.4: + + ```shell + brew tab eddies/spark-tap + brew install apache-spark@2.4.6 + ``` + + +### Python support + +In order to run Polynote, you'll need to have a working Python 3 environment (including `pip`) installed. Refer to +[Python's installation instructions](https://wiki.python.org/moin/BeginnersGuide/Download){:target="_blank"} for +instructions. + +!!! tip "Using Python with Polynote" + Polynote officially supports **Python 3.7**. This is the version of Python supported by the Spark versions we're + targeting. + + _Users have reported getting it to work with Python 3.6 and Python 3.8 (without Spark) with minimal effort._ + +You'll also need to install Polynote's Python dependencies: + +``` +pip install -r ./requirements.txt +``` + +## Download +Polynote consists of a JVM-based server application, which serves a web-based client. To try it locally, find the latest +release on the [releases page](https://github.com/polynote/polynote/releases){:target="_blank"} and download the attached +`polynote-dist.tar.gz` file (you'll find it under `Assets`). Unpack the archive: + +```shell +tar -zxvpf polynote-dist.tar.gz +cd polynote +``` + +## Configure +You won't need to change any of the default configuration in order to get Polynote up and running locally. + +Please see the [configuration](configuration.md) documentation for more details on configuring Polynote. + +## Run +To start the server, run the included python script: + +``` +./polynote.py +``` + +Once the server has started, navigate your browser to `http://localhost:8192` (if using the default network configuration). diff --git a/docs/docs/kernel-pane.md b/docs/docs/kernel-pane.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/mixing-programming-languages.md b/docs/docs/mixing-programming-languages.md new file mode 100644 index 0000000..4168da7 --- /dev/null +++ b/docs/docs/mixing-programming-languages.md @@ -0,0 +1,215 @@ +One of Polynote's most interesting features is its support for polyglot notebooks, where cells within the same notebook +can be written in different languages. + +Variables can also be shared across languages, as all execution is done within a single process. + +### Execution state in Polynote + +We hinted [previously](basic-usage.md#the-symbol-table-and-input-scope) that the way Polynote handles cell execution +is a little special. + +As a reminder, the kernel keeps track of all the symbols defined by a cell execution. These symbols are part +of the cell's state, which is made available to downstream cells (those below the cell in question) when they in turn +are executed. + +Polynote stores these symbols, alongside their types and other information, in a Scala-based format. Symbols defined by +and provided to other languages are wrapped (and unwrapped) appropriately. + +!!!info "" + For more details, how this works under the hood, and information about its limitations and caveats, check out + the [advanced documentation](advanced-polyglot.md). + +### Polyglot example + +Continuing our example notebook from [before](basic-usage.md), here's a trivial example (note that the bottom cell has +Python selected in the language drop-down). + +![Accessing a Scala variable from Python](images/simple-py-interop.png) + +The `foo` and `bar` Scala variables can be accessed directly in Python, because they represent primitive +types (`:::scala Int`s in this case). This also works the other way 'round, from Python to Scala, in much the same way. + +### Use Case: Plotting Scala data with Matplotlib + +!!!tip "Real-world usage at Netflix" + This example notebook illustrates a contrived version of a common use-case at Netflix. [Our recommendation and + personalization infrastructure is heavily JVM-based](https://www.slideshare.net/FaisalZakariaSiddiqi/ml-infra-for-netflix-recommendations-ai-nextcon-talk), + and Algorithm Engineers and Scientists want to interface with our platform in order to, for example: + + - [transform data in a type-safe way using Scala](https://portal.klewel.com/watch/webcast/scala-days-2019/talk/12/) + - [access Model Lifecycle data](https://www.usenix.org/conference/opml20/presentation/cepoi) + - [generate features based on online data snapshots](https://netflixtechblog.com/distributed-time-travel-for-feature-generation-389cccdd3907) + + Once they get the data, they'd like to analyze and visualize it using the Python tools they are familiar with, such + as [pandas](https://pandas.pydata.org/) and [matplotlib](https://matplotlib.org/). + +A more complex example of a more typical usage can be found in the `examples/` folder in +[`Plotting Scala data with Matplotlib.ipynb`](https://github.com/polynote/polynote/blob/master/docs/examples/Plotting%20Scala%20data%20with%20Matplotlib.ipynb) +which is reproduced below. + +We'll go over this example in detail. + +![Plotting Scala data with matplotlib through Python](images/plotting-scala-matplotlib.png) + +#### A direct approach using primitives + +This notebook takes a direct approach to polyglot development. By crossing language boundaries using primitive data +structures, you don't need to worry about the details of language-specific data structures or implementations. + +Let's go over this notebook, which consists of two cells. The first, a Scala cell, prepares some data, while the second, +a Python cell, plots it. + +!!!example "Scala cell" + ```scala + val colors = Array("red", "green", "blue") + val n = 750 + val flattenedLength = colors.length * n * 2 // 2 for x and y + val data = (0 until flattenedLength).map(_ => Math.random()).toArray // converted to Array for Python interop + ``` + +This cell generates an `:::scala Array[Double]` that is a flattened representation of data in three dimensions - `color`, +`x`, and `y`. + +This data is then fed into the Python cell to be plotted: + +!!!example "Python cell" + ```python + import matplotlib.pyplot as plt + from numpy.random import rand + + fig, ax = plt.subplots() + + for color in colors: + # since data is just a 1-D array we need to split it up to fit the matplotlib example + x = data[:n] + data = data[n:] + y = data[:n] + data = data[n:] + + scale = 200.0 * rand(n) + ax.scatter(x, y, c=color, s=scale, label=color, + alpha=0.3, edgecolors='none') + + ax.legend() + ax.grid(True) + + plt.show() + ``` + +This cell steps through the `data` array and, for each `color`, unflattens the `x` and `y` data for plotting (note that +both `data` and `color` are Scala variables accessed directly from Python). + +#### Can we do better? + +This is pretty straightforward, but you might have noticed that the interchange values between the two languages are +`:::scala Array`s, rather than the more idiomatic (for Scala) `:::scala Seq`s. You might also notice that it was pretty +unnatural to flatten `data` like we did. + +The reason for this is that it's easier to interchange between Scala and Python using primitives and basic structures +like Arrays. + +!!!info "" + You might not realize it, but when you work with Scala there's all sorts of stuff going on behind the scenes, + such as implicits, or encoded Java method names (like `$colon$colon()`). + + Python doesn't know about this, so when we interact with a Scala object from Python we see it in all its gory details: + + ![What a Seq looks like when viewed from Python](images/python-scala-seq.png){: width=50% } + +However, with a little Scala knowledge we can rewrite the plotting code to be more idiomatic, at the expense of having +to do (a bit) more work in Python to massage the data. + +#### A more idiomatic implementation + +First, we'll use some more idiomatic code to generate our point data: + +```scala +case class Point(x: Double, y: Double, color: String) +val colors = Seq("red", "green", "blue") +val n = 750 +val idiomaticData = for { + color <- colors + n <- (0 to n) +} yield Point(Math.random(), Math.random(), color) +``` + +Wow, a `:::scala case class` AND a `:::scala for` expression? This is surely extremely idiomatic Scala code. +Not an `:::scala Array` in sight! + +Let's inspect `idiomaticData` for a moment. + +![Inspecting the value of idiomaticData using Polynote's inspection tools](images/inspecting-idiomaticData.gif) + +!!!info "Inspecting and Visualizing Data" + This image showcases using the [Quick Inspector](symbol-table.md#quick-inspector) and [Viz cells](visualization-and-viz-cells.md) + to easily inspect values within a notebook. + +Now that we've taken a look at `idiomaticData` and seen that it's a `:::scala Seq[Point]`, we can delete the Viz cell and +start working with it using Python. + +Since we're no longer working with primitives, we need to do a little data preparation. We'll build a `:::python list`, +iterate over `idiomaticData`, and extract values from `Point`. + +We want to end up with a simple 2D-Array we can use with `matplotlib`. + +```python +data = [] +for idx in range(0, idiomaticData.length()): + point = idiomaticData.apply(idx) + data.append([point.x(), point.y(), point.color()]) +``` + +There are two things of note here. + +First, we need to manually iterate through the collection, generating an index which we can pass into the +[`:::scala apply(idx: Int): A`](https://www.scala-lang.org/api/2.12.0/scala/collection/Seq.html#apply(idx:Int):A) +method of the `Seq` to select elements from it. + +Second, we'll access the `:::scala case class` members using the member accessor methods as usual, but note that with +Python we need to specify `()`, as in `:::python point.x()`, since of course in Python parentheses are always required, +even for nullary functions (ones that have no arguments). + +At this point, the `data` variable is a nice `:::python list` of `:::python list`s, and we can easily turn it into a +Pandas DataFrame: +```python +import pandas as pd + +df = pd.DataFrame(data=data, columns=['x', 'y', 'color']) +``` + +Note that we have to manually specify the column names here. A more advanced example might extract the fields from +`:::scala Point` and pass them into Python, but we'll pass on that for now. + +Now that our data is a Pandas DataFrame we can manipulate it as usual. For example, we can add a `scale` column to it: + +```python +import numpy as np + +df['scale'] = np.random.uniform(1, 200, df.shape[0]) +``` + +Finally, we can plot our DataFrame in the familiar way. + +```python +import matplotlib.pyplot as plt + +fig, ax = plt.subplots() + +groups = df.groupby("color") +for name, group in groups: + ax.scatter(group.x, group.y, c=group.color, s=group.scale, label=name, + alpha=0.3, edgecolors='none') + +ax.legend(loc="upper right") +ax.grid(True) + +plt.show() +``` + +That's it! This notebook can be found in the `examples/` folder as well, in +[`Plotting Scala data with Matplotlib (Idiomatic).ipynb`](https://github.com/polynote/polynote/blob/master/docs/examples/Plotting%20Scala%20data%20with%20Matplotlib%20(Idiomatic).ipynb). +You can see a screenshot below. + +![Plotting Scala data with matplotlib through Python, idiomatically](images/plotting-scala-matplotlib-idiomatic.png) + +Next, read about [using Spark with Polynote](spark.md). \ No newline at end of file diff --git a/docs/docs/notebook-configuration.md b/docs/docs/notebook-configuration.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/notebook.md b/docs/docs/notebook.md new file mode 100644 index 0000000..1c33723 --- /dev/null +++ b/docs/docs/notebook.md @@ -0,0 +1 @@ +blah blah \ No newline at end of file diff --git a/docs/docs/notebooks-pane.md b/docs/docs/notebooks-pane.md new file mode 100644 index 0000000..8a68b3c --- /dev/null +++ b/docs/docs/notebooks-pane.md @@ -0,0 +1,52 @@ +The Notebooks Pane displays a listing of all the notebooks in your current workspace. + +![Notebooks Pane](images/notebooks-pane.png){: .centered-image } + +Notebooks are displayed in a directory structure. + +Clicking on a notebook opens it, and clicking on a directory expands or collapses it. + +!!!tip + The current workspace can be set in the [Storage Configuration](configuration.md#storage). + +### Creating a new notebook + +You can create a new notebook by clicking on the green plus sign +![green plus sign](images/green-plus.png){: .inline-image} in the corner. + +This will bring up the Create Notebook Dialog, where you can enter the notebook name. If a notebook with that name +already exists, a number will be appended at the end of the filename. + +!!!tip + Any slashes in the notebook name will place your new notebook in a directory (creating them if they don't already + exist). + + Addiitonally, Polynote will add an extension if you don't specify one. + + For example, typing in `foo/bar/My Notebook` will create a `My Notebook.ipynb` file and put it into a directory + named `bar`, within a directory named `foo`. + + +### Importing a notebook + +Importing a notebook into Polynote is as simple as dragging and dropping the file onto the Notebooks Pane! + +![Dragging and Dropping a Notebook](images/drag-drop-notebook.gif) + +!!!info "Supported import formats" + Polynote supports importing your existing Jupyter `.ipynb` files and Zeppelin `.json` files. Zeppelin files will be + automatically converted into `.ipynb` files upon import. + + Note that Polynote doesn't do any translation of the notebook contents, so any platform-specific features, such as + Jupyter "magics" (like Toree's `%AddJar`) or Zeppelin's `z.load` syntax will need to be manually converted. + +### Rename, Copy, Delete + +You can Rename, Copy, and Delete notebooks using the Context Menu, which you can bring up by right-clicking on a +notebook. + +Here's a demonstration of Renaming a notebook using the Context Menu. The Copy and Delete functions work rather +similarly. + +![Renaming notebook using Context Menu](images/context-menu-rename.gif) + diff --git a/docs/docs/spark.md b/docs/docs/spark.md new file mode 100644 index 0000000..8bc8cb5 --- /dev/null +++ b/docs/docs/spark.md @@ -0,0 +1,11 @@ +# Using Spark with Polynote + +Polynote has deep integration with [Apache Spark](https://spark.apache.org), and supports running both Scala and Python +code with Spark out of the box. + +In order to use Spark, you must first have it [installed](installation.md#spark-support). Then, open the +[configuration](notebook-configuration.md) for the notebook you'd like to use with Spark, and scroll down to the +`Spark configuration` section. As long as you set anything there - any Spark property, any [Spark template](configuration.md#spark) +- and Polynote will launch your notebook with Spark enabled. + +![spark-master-config](images/spark-master-config.png) \ No newline at end of file diff --git a/docs/docs/text-cells.md b/docs/docs/text-cells.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/toolbar.md b/docs/docs/toolbar.md new file mode 100644 index 0000000..55f74a1 --- /dev/null +++ b/docs/docs/toolbar.md @@ -0,0 +1,66 @@ +The toolbar is one of the primary ways you'll be interacting with Polynote. + +It is divided into sections, which we will go over now. + +![Toolbar Sections](images/toolbar-sections.png) + +The toolbar is divided into three sections: [Notebook](#notebook-section){: style="color: red" }, +[Cell](#cell-section){: style="color: blue" } and [About](#about-section){: style="color: purple" }. + +### Notebook section + +![Notebook Section](images/toolbar-notebook-section.png) + +This section contains buttons that interact with the currently active notebook. The three relevant buttons are: + +- The Run All button ![Run All Cells](images/run-all-cells.png){: .inline-image } which queues up all cells of the current notebook +to run in order. +- The Download button ![Download Notebook](images/download-current-notebook.png){: .inline-image } which downloads the current notebook as +an ipynb file. +- The Clear Output button ![Clear Output](images/clear-output.png){: .inline-image } which clears the output of all notebook cells. + +The other two buttons are placeholders for future functionality that are currently not in use. + +### Cell section + +This section contains elements for interacting with the currently active cell. + +This section has two subsections. On the left is the Cell section, which holds elements for interacting with the +current cell regardless of its type. On the right is the a cell detail section, which changes depending on whether the +currently active cell is a Code or Text cell. + +We'll start with the invariant Cell section first. + +![Toolbar Cell Section](images/toolbar-cell-section.png) + +- The Cell Type dropdown is used to change the type of the cell. This is equivalent to the dropdown found on + [Code cells](code-cells.md) but it can also convert a cell from `Code` to `Text` and vice-versa. +- The Insert Cell Above ![Insert Above](images/toolbar-cell-insert-above.png){: .inline-image } and Below + ![Insert Below](images/toolbar-cell-insert-below.png){: .inline-image } buttons, which insert a new cell above or + below the currently selected cell. +- The Delete Cell button ![Delete Cell](images/toolbar-cell-delete.png){: .inline-image } which deletes the currently + selected cell. + +#### Code Cells + +![Toolbar Code Cell](images/toolbar-code-cell.png) + +- The run cell button ![Run Cell](images/toolbar-code-run.png){: .inline-image } runs the currently selected cell. +- The next button ![Run Cell Until Selected](images/toolbar-code-run-until-selected.png){: .inline-image } runs all + cells above the currently selected cell until (and including) the currently selected cell. +- The stop button ![Stop Execution](images/toolbar-code-stop.png){ .inline-image } cancels any currently-running + execution (not limited to the current cell). + +#### Text Cells + +![Toolbar Text Cell](images/toolbar-text-cell.png) + +The Text cell subsection contains the typical rich text editor interaction elements, along with a few special ones. + +For more details, see the [Text Cell](text-cells.md) documentation. + +### About section + +![Toolbar About](images/toolbar-about.png){: .centered-image } + +Finally, the About section at the top-right corner contains buttons to access the [About menu](about-menu.md). \ No newline at end of file diff --git a/docs/docs/tour.md b/docs/docs/tour.md new file mode 100644 index 0000000..dacddeb --- /dev/null +++ b/docs/docs/tour.md @@ -0,0 +1,47 @@ +Welcome to the tour of Polynote's UI! + +Polynote takes its inspiration from IDE-like tools, bringing a rich UI into the notebook experience. + +### The Polynote UI + +Here, we'll go overthe various parts of the UI briefly, and then drill down on each later in the tour. + +![The Polynote UI with sections labelled](images/polynote-ui-labelled.png) + +This screenshot divides the UI into four sections: + +- The [Toolbar](toolbar.md){: style="color:green"} at the top, which collects buttons for interaction with the notebook +- The [Notebooks List](notebooks-list.md){: style="color:darkorange"} pane on the left, a file-like browser showing the + notebooks in Polynote's workspace +- The [Kernel Pane](kernel-pane.md){: style="color:purple"} on the right, which shows the current state of the notebook. +- The [Notebook](notebook.md){: style="color:red"} section itself in the middle, which displays the actual notebook. + +The above sections provide details about the specific functionality of each part of the UI. + +### Panes in Polynote + +Before you go learn more about the specifics, let's just talk about how the two panes work. + +#### Panes can be collapsed + +If you'd like more horizontal space to see the [Notebook](notebook.md) section, you can collapse one or both of the +panes. + +Simply click on any empty space on the header, and the pane will collapse to the side. Click the title of the header +(for example, `Kernel` if you have collapsed the [Kernel Pane](kernel-pane.md)) in order to expand the pane again. + +![Collapsing and Expanding a Pane](images/collapse-pane.gif){: width=50% } + +#### Panes can be resized + +If you want more room for the [Notebook](notebook.md) but don't want to hide a pane entirely, you can resize a pane by +dragging it. + +![Resizing a Pane](images/resize-pane.gif) + +!!!tip + Polynote remembers your pane collapse and size preferences in your browser's local storage. + + As long as you're using Polynote with the same browser, your customizations should be persisted! + +Next up, learn about the [Toolbar](toolbar.md). \ No newline at end of file diff --git a/docs/docs/visualization-and-viz-cells.md b/docs/docs/visualization-and-viz-cells.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/favicon.ico b/docs/favicon.ico new file mode 100644 index 0000000..2c1f857 Binary files /dev/null and b/docs/favicon.ico differ diff --git a/images/autocomplete.png b/docs/images/autocomplete.png similarity index 100% rename from images/autocomplete.png rename to docs/images/autocomplete.png diff --git a/docs/images/code-cell.png b/docs/images/code-cell.png deleted file mode 100644 index b113273..0000000 Binary files a/docs/images/code-cell.png and /dev/null differ diff --git a/images/compile-errors.png b/docs/images/compile-errors.png similarity index 100% rename from images/compile-errors.png rename to docs/images/compile-errors.png diff --git a/images/data-table.png b/docs/images/data-table.png similarity index 100% rename from images/data-table.png rename to docs/images/data-table.png diff --git a/images/equations.png b/docs/images/equations.png similarity index 100% rename from images/equations.png rename to docs/images/equations.png diff --git a/docs/images/fresh-home-screen.png b/docs/images/fresh-home-screen.png deleted file mode 100644 index 196036f..0000000 Binary files a/docs/images/fresh-home-screen.png and /dev/null differ diff --git a/images/heart-solid.svg b/docs/images/heart-solid.svg similarity index 100% rename from images/heart-solid.svg rename to docs/images/heart-solid.svg diff --git a/images/hero-animation.gif b/docs/images/hero-animation.gif similarity index 100% rename from images/hero-animation.gif rename to docs/images/hero-animation.gif diff --git a/images/highlight-running.png b/docs/images/highlight-running.png similarity index 100% rename from images/highlight-running.png rename to docs/images/highlight-running.png diff --git a/images/logo.svg b/docs/images/logo.svg similarity index 100% rename from images/logo.svg rename to docs/images/logo.svg diff --git a/docs/images/logo_dark.svg b/docs/images/logo_dark.svg new file mode 100644 index 0000000..4c2453d --- /dev/null +++ b/docs/images/logo_dark.svg @@ -0,0 +1,51 @@ + + diff --git a/docs/images/logo_small.svg b/docs/images/logo_small.svg new file mode 100644 index 0000000..6d6615c --- /dev/null +++ b/docs/images/logo_small.svg @@ -0,0 +1,39 @@ + + diff --git a/images/netflix-oss.svg b/docs/images/netflix-oss.svg similarity index 100% rename from images/netflix-oss.svg rename to docs/images/netflix-oss.svg diff --git a/images/ordered.png b/docs/images/ordered.png similarity index 100% rename from images/ordered.png rename to docs/images/ordered.png diff --git a/images/plots.png b/docs/images/plots.png similarity index 100% rename from images/plots.png rename to docs/images/plots.png diff --git a/docs/images/results.png b/docs/images/results.png deleted file mode 100644 index ac3f578..0000000 Binary files a/docs/images/results.png and /dev/null differ diff --git a/images/rich-data.png b/docs/images/rich-data.png similarity index 100% rename from images/rich-data.png rename to docs/images/rich-data.png diff --git a/images/rich-text.png b/docs/images/rich-text.png similarity index 100% rename from images/rich-text.png rename to docs/images/rich-text.png diff --git a/docs/images/spark-master-config.png b/docs/images/spark-master-config.png deleted file mode 100644 index ca8ecb3..0000000 Binary files a/docs/images/spark-master-config.png and /dev/null differ diff --git a/images/spark.png b/docs/images/spark.png similarity index 100% rename from images/spark.png rename to docs/images/spark.png diff --git a/images/symbol-table.png b/docs/images/symbol-table.png similarity index 100% rename from images/symbol-table.png rename to docs/images/symbol-table.png diff --git a/images/tasks.png b/docs/images/tasks.png similarity index 100% rename from images/tasks.png rename to docs/images/tasks.png diff --git a/docs/images/text-editing.png b/docs/images/text-editing.png deleted file mode 100644 index 88e7519..0000000 Binary files a/docs/images/text-editing.png and /dev/null differ diff --git a/docs/index.md b/docs/index.md index 17473f9..c1af09d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,10 +1,4 @@ --- -title: Documentation -layout: docs -permalink: /docs/ +template: home.html +title: Home --- - -# Polynote Documentation - -- [User guide](01-installation.md) -- Developer documentation (TODO) \ No newline at end of file diff --git a/docs/overrides/home.html b/docs/overrides/home.html new file mode 100644 index 0000000..f0fd2f1 --- /dev/null +++ b/docs/overrides/home.html @@ -0,0 +1,131 @@ +{% extends "./main.html" %} +{% block tabs %} +{{ super() }} + +

The polyglot notebook
with first-class Scala support.

+ Learn to + + + + your notebooks again. +

Try it!

Features

+ Polynote is a different kind of notebook. It supports mixing multiple languages in one notebook, and sharing + data between them seamlessly. It encourages reproducible notebooks with its immutable data model. (Learn more) +

+ Here are some of its unique features: +

+ +

Editing

Interactive auto-complete helps you find what you're looking for without switching to the documentation.

Highlighting your errors helps you quickly figure out what's gone wrong.

Rich text editing lets you see your formatted text cells as you write them, and edit them like a document.

Easily insert LaTeX equations into your text cells.

Executing

Always know what the kernel is doing with individual task tracking.

The symbol table keeps track of what you've defined, and what's available in the current cell. Inspect any value for detailed information and rich visualizations.

See exactly what code is executing right now, as Polynote highlights the running statement in real-time.

Ordered cell semantics ensures your notebook can be reproducibly executed.

Data and visualization

First-class integration with Apache Spark™ for exploring, analyzing, and visualizing big data.

Browse table-structured data, including collections of data structures and Spark data sets.

See rich representations of your data, automatically derived or supplied by a typeclass.

Easily create visualizations with the built-in plot editor, or define your own with the Vega visualization language.

+{% endblock %} + +{% block content %}{% endblock %} +{% block footer %}{% endblock %} diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 0000000..484b9cb --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} + +{% block libs %} + {{ super() }} + +{% endblock %}} + +{% block scripts %} + + {{ super ()}} +{% endblock %} \ No newline at end of file diff --git a/polynote.svg b/docs/polynote.svg similarity index 100% rename from polynote.svg rename to docs/polynote.svg diff --git a/docs/polynote_small.svg b/docs/polynote_small.svg new file mode 100644 index 0000000..6d6615c --- /dev/null +++ b/docs/polynote_small.svg @@ -0,0 +1,39 @@ + + diff --git a/style/style.css b/docs/style/style.css similarity index 69% rename from style/style.css rename to docs/style/style.css index a2cd18b..4327dd8 100644 --- a/style/style.css +++ b/docs/style/style.css @@ -1,93 +1,12 @@ @import url('https://fonts.googleapis.com/css?family=Raleway:400,400i,700,700i&display=swap'); -html { - padding: 0; - margin: 0; - background: black; -} - body { - background: white; - padding: 0; - margin: 0; - font-family: 'Raleway', Helvetica, Arial, sans-serif; font-size: 12pt; line-height: 1.4; - color: #333333; -} - -a { - color: #516a84; -} - -header { - height: 4em; - background: black; - background: linear-gradient(to bottom, #444444, black); -} - -header h1 { - margin: 0; - padding: 0; - font-size: 200%; - line-height: 2; - vertical-align: middle; -} - -header h1 a { - text-decoration: none; - display: inline-block; - height: 1.25em; - line-height: 1.25; - vertical-align: middle; -} - -header h1 img { - height: 1.25em; - line-height: 1.25; - display: inline-block; -} - -header nav { - position: absolute; - right: 0; - top: .5em; - line-height: 3.5em; -} - -header nav ul { - list-style: none; - margin: 0; - padding: 0; -} - -header nav ul li { - display: inline-block; - padding: 0; -} - -header nav ul li a { - color: white; - text-decoration: none; - text-transform: uppercase; - padding: 1.5em; - font-size: 83.3%; - line-height: 1; -} - -header nav ul li a:hover { - background: white; - color: black; -} - -section { - max-width: 68em; - margin: 0 auto; - position: relative; } -#Home { - +.home { + font-family: 'Raleway', Helvetica, Arial, sans-serif; } #Home #Splash { @@ -265,22 +184,6 @@ section.page-content, section.post-content { max-width: 100%; } -.warning { - background-color: #fff8c4; - border: 1px solid #f7deae; - border-radius: .5em; - padding: 2em; - margin: 3em auto; - max-width: 50em; -} - -.warning p:last-child { margin-bottom: 0 } - -.warning h2 { - margin-top: 0; - font-size: 140%; -} - pre.highlight { background: #f3f3f3; border: 1px solid #cccccc; @@ -295,35 +198,53 @@ p code { background-color: #f3f3f3; } -footer { - min-height: 4em; - padding: 2em; - background: black; - /*background: linear-gradient(to bottom, #444444, black);*/ - margin-top: 8em; - color: white; +.md-tabs__link { + margin-top: 0.4rem; } -footer section { - display: grid; - grid-template-columns: 1fr 1fr; +.md-tabs__item { + height: 2rem; } -footer a { - color: white; +.md-logo img { + width: unset !important; + height: 2rem !important; +} + +/* Hide the title since the logo already says 'Polynote' */ +.md-header__title { + visibility: hidden; +} + +.num { + font-size: x-large; + line-height: 0; + vertical-align: middle; + color: red; +} + +.md-header { + z-index: 10; /* header should float above. */ +} + +.ff-overlay { + z-index: 2 !important; /* the freezeframe library CSS has a z-index of 100 for some reason which messes with other things */ } -footer nav { - font-size: 11pt; +.centered-image, .ff-image { + /* center small images on screen */ + display: block; + margin-left: auto; + margin-right: auto; } -footer nav ul { - list-style: none; - column-count: 2; - margin: 0; - padding: 0; +.ff-canvas { + /* center the freezeframe canvas to match the image*/ + right: 0; + margin: auto; } -footer nav ul li { - padding: .5em 0; +.inline-image { + height: 1.5em !important; + vertical-align: middle; } \ No newline at end of file diff --git a/index.md b/index.md deleted file mode 100644 index 1ec2ab8..0000000 --- a/index.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: home -title: Polynote | The polyglot Scala notebook ---- \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..6f46455 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,67 @@ +site_name: Polynote +extra_css: + - style/style.css +plugins: + - search + - section-index +markdown_extensions: + - toc: + permalink: true + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.highlight + - pymdownx.inlinehilite + - pymdownx.keys + - pymdownx.tabbed + - pymdownx.magiclink + - footnotes + - attr_list +repo_url: https://github.com/polynote/polynote +repo_name: polynote +nav: + - index.md + - Docs: + - docs/index.md + - Installation: docs/installation.md + - Basic Usage: docs/basic-usage.md + - Mixing Programming Languages: docs/mixing-programming-languages.md + - Using Spark with Polynote: docs/spark.md + - Tour: + - docs/tour.md + - Toolbar: docs/toolbar.md + - Notebooks Pane: docs/notebooks-list.md + - Kernel Pane: docs/kernel-pane.md + - Notebook: + - docs/notebook.md + - Text Cells: docs/text-cells.md + - Code Cells: docs/code-cells.md + - Visualization and Viz Cells: docs/visualization-and-viz-cells.md + - Notebook Configuration: docs/notebook-configuration.md + - About Menu: docs/about-menu.md + - FAQ: docs/faq.md + - Advanced: + - Server Configuration: docs/configuration.md + - Scala / Python Interop: docs/advanced-polyglot.md + - Using Polynote with Docker: docs/docker.md + - Developing with Polynote: docs/development.md + - About: about.md + - Contribute: contribute.md +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/polynote/polynote + - icon: fontawesome/brands/gitter + link: https://gitter.im/polynote/polynote +theme: + name: material + logo: images/logo_dark.svg + favicon: favicon.ico + custom_dir: docs/overrides + palette: + primary: black + features: + - navigation.instant + - navigation.tabs + - navigation.expand + diff --git a/news.md b/news.md deleted file mode 100644 index cf3afda..0000000 --- a/news.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -layout: default -title: Polynote News -permalink: /news/ ---- - -{%- for post in site.posts -%} - -{%- assign date_format = site.minima.date_format | default: "%b %-d, %Y" -%} -{{ post.date | date: date_format }} -

- - {{ post.title | escape }} - -

-{{ post.excerpt }} - -{%- endfor -%} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4bb81f5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +mkdocs-material +pygments +section-index diff --git a/style/code.css b/style/code.css deleted file mode 100644 index 596e408..0000000 --- a/style/code.css +++ /dev/null @@ -1,209 +0,0 @@ -.highlight table td { padding: 5px; } -.highlight table pre { margin: 0; } -.highlight .cm { - color: #999988; - font-style: italic; -} -.highlight .cp { - color: #999999; - font-weight: bold; -} -.highlight .c1 { - color: #999988; - font-style: italic; -} -.highlight .cs { - color: #999999; - font-weight: bold; - font-style: italic; -} -.highlight .c, .highlight .ch, .highlight .cd, .highlight .cpf { - color: #999988; - font-style: italic; -} -.highlight .err { - color: #a61717; - background-color: #e3d2d2; -} -.highlight .gd { - color: #000000; - background-color: #ffdddd; -} -.highlight .ge { - color: #000000; - font-style: italic; -} -.highlight .gr { - color: #aa0000; -} -.highlight .gh { - color: #999999; -} -.highlight .gi { - color: #000000; - background-color: #ddffdd; -} -.highlight .go { - color: #888888; -} -.highlight .gp { - color: #555555; -} -.highlight .gs { - font-weight: bold; -} -.highlight .gu { - color: #aaaaaa; -} -.highlight .gt { - color: #aa0000; -} -.highlight .kc { - color: #000000; - font-weight: bold; -} -.highlight .kd { - color: #000000; - font-weight: bold; -} -.highlight .kn { - color: #000000; - font-weight: bold; -} -.highlight .kp { - color: #000000; - font-weight: bold; -} -.highlight .kr { - color: #000000; - font-weight: bold; -} -.highlight .kt { - color: #445588; - font-weight: bold; -} -.highlight .k, .highlight .kv { - color: #000000; - font-weight: bold; -} -.highlight .mf { - color: #009999; -} -.highlight .mh { - color: #009999; -} -.highlight .il { - color: #009999; -} -.highlight .mi { - color: #009999; -} -.highlight .mo { - color: #009999; -} -.highlight .m, .highlight .mb, .highlight .mx { - color: #009999; -} -.highlight .sb { - color: #d14; -} -.highlight .sc { - color: #d14; -} -.highlight .sd { - color: #d14; -} -.highlight .s2 { - color: #d14; -} -.highlight .se { - color: #d14; -} -.highlight .sh { - color: #d14; -} -.highlight .si { - color: #d14; -} -.highlight .sx { - color: #d14; -} -.highlight .sr { - color: #009926; -} -.highlight .s1 { - color: #d14; -} -.highlight .ss { - color: #990073; -} -.highlight .s, .highlight .sa, .highlight .dl { - color: #d14; -} -.highlight .na { - color: #008080; -} -.highlight .bp { - color: #999999; -} -.highlight .nb { - color: #0086B3; -} -.highlight .nc { - color: #445588; - font-weight: bold; -} -.highlight .no { - color: #008080; -} -.highlight .nd { - color: #3c5d5d; - font-weight: bold; -} -.highlight .ni { - color: #800080; -} -.highlight .ne { - color: #990000; - font-weight: bold; -} -.highlight .nf, .highlight .fm { - color: #990000; - font-weight: bold; -} -.highlight .nl { - color: #990000; - font-weight: bold; -} -.highlight .nn { - color: #555555; -} -.highlight .nt { - color: #000080; -} -.highlight .vc { - color: #008080; -} -.highlight .vg { - color: #008080; -} -.highlight .vi { - color: #008080; -} -.highlight .nv, .highlight .vm { - color: #008080; -} -.highlight .ow { - color: #000000; - font-weight: bold; -} -.highlight .o { - color: #000000; - font-weight: bold; -} -.highlight .w { - color: #bbbbbb; -} -.highlight { - background-color: #f8f8f8; -}

404

The polyglot notebookwith first-class Scala support.

- Learn to - - - - love - - - - your notebooks again. -

Try it!

Features

Editing

Executing

Data and visualization

{{ page.title }}

The polyglot notebookwith first-class Scala support.

- Learn to - - - - love - - - - your notebooks. -

Try it!

Features

Editing

Executing

Data and visualization

{{ page.title }}

The polyglot notebookwith first-class Scala support.

+ Learn to + + + + love + + + + your notebooks again. +

Try it!

Features

Editing

Executing

Data and visualization

- - {{ post.title | escape }} - -

The polyglot notebook
with first-class Scala support.

- Learn to - - - - your notebooks again. -

The polyglot notebook
with first-class Scala support.

- Learn to - - - - your notebooks. -

The polyglot notebook
with first-class Scala support.

+ Learn to + + + + your notebooks again. +