Update docs and examples

.dockerignore

@@ -0,0 +1,2 @@
+dev/
+dist/

README.md

@@ -34,13 +34,12 @@ wh1isper/sparglim-server
 
 Access `http://localhost:4040` for Spark-UI and `sc://localhost:15002` for Spark Connect Server. [Use sparglim to setup SparkSession to connect to Spark Connect Server](#connect-to-spark-connect-server).
 
-## Install
+## Install: `pip install sparglim[all]`
 
 - Install only for config and daemon spark connect server `pip install sparglim`
-- Install for pyspark app `pip install sparglim["pyspark"]`
-- Install for using magic within ipython/jupyter (will also install pyspark) `pip install sparglim["magic"]`
-- Install for pyspark app on k8s `pip install sparglim["k8s"]`
-- Install for all above (such as using magic in jupyterlab on k8s) `pip install sparglim["all"]`
+- Install for pyspark app `pip install sparglim[pyspark]`
+- Install for using magic within ipython/jupyter (will also install pyspark) `pip install sparglim[magic]`
+- **Install for all above** (such as using magic in jupyterlab on k8s) `pip install sparglim[all]`
 
 ## Feature
 
@@ -134,19 +133,29 @@ Query the view by `%SQL`:
 %sql SELECT * FROM tb
 ```
 
-or
+`%SQL` result dataframe can be assigned to a variable:
+
+```ipython
+df = %sql SELECT * FROM tb
+df
+```
+
+or `%%SQL` can be used to execute multiple statements:
 
 ```ipython
 %%sql SELECT
         *
         FROM
-        tb
+        tb;
 ```
 
 You can also using Spark SQL to load data from external data source, such as:
 
 ```ipython
-%sql CREATE TABLE tb_people USING json OPTIONS (path "/path/to/file.json")
+%%sql CREATE TABLE tb_people
+USING json
+OPTIONS (path "/path/to/file.json");
+Show tables;
 ```
 
 ## Develop

dev/jupyter-sparglim-on-k8s/Dockerfile.jupyterlab-sparglim → dev/docker/Dockerfile.jupyterlab-sparglim

@@ -13,4 +13,4 @@ RUN pip install --no-cache-dir ./sparglim[all]
 ENTRYPOINT ["tini","--","jupyter-lab", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--IdentityProvider.token=''"]
 EXPOSE 8888
 
-# docker build -t wh1isper/jupyterlab-sparglim:dev -f dev/jupyter-sparglim-on-k8s/Dockerfile.jupyterlab-sparglim . && ./dev/jupyter-sparglim-on-k8s/reload.sh
+# docker build -t wh1isper/jupyterlab-sparglim:dev -f dev/docker/Dockerfile.jupyterlab-sparglim . && ./scripts/reload.sh

dev/sparglim-server/Dockerfile.sparglim-server → dev/docker/Dockerfile.sparglim-server

@@ -6,4 +6,4 @@ RUN pip install --no-cache-dir ./sparglim
 ENTRYPOINT [ "tini" ,"--"]
 CMD [ "sparglim-server", "start" ]
 
-# docker build -t wh1isper/sparglim-server:dev -f dev/sparglim-server/Dockerfile.sparglim-server . && ./dev/sparglim-server/reload.sh
+# docker build -t wh1isper/sparglim-server:dev -f dev/docker/Dockerfile.sparglim-server . && ./dev/scripts/reload.sh

dev/jupyter-sparglim-sc/k8s/jupyter-sparglim/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sparglim-app
+  namespace: sparglim
+  labels:
+    app: sparglim
+spec:
+  replicas: 1
+
+  selector:
+    matchLabels:
+      app: sparglim-app
+  template:
+    metadata:
+      labels:
+        app: sparglim-app
+    spec:
+      containers:
+        - name: sparglim-app
+          image: wh1isper/jupyterlab-sparglim:dev
+          ports:
+            - containerPort: 8888
+          imagePullPolicy: Never
+          env:
+            - name: SPARGLIM_SQL_MODE
+              value: connect_client
+            - name: SPARGLIM_REMOTE
+              value: "sc://sparglim-server-headless-service:15002"
+            - name: SPARGLIM_DRIVER_HOST
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: SPARGLIM_DRIVER_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name

examples/jupyter-sparglim-sc/k8s/lab-service.yaml → dev/jupyter-sparglim-sc/k8s/jupyter-sparglim/lab-service.yaml

@@ -10,11 +10,6 @@ spec:
       port: 8888
       nodePort: 30888
       targetPort: 8888
-    - name: spark-ui
-      protocol: TCP
-      port: 4040
-      nodePort: 30040
-      targetPort: 4040
   selector:
     app: sparglim-app
   type: NodePort

dev/jupyter-sparglim-sc/k8s/sparglim-server/connect-server-service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sparglim-server-service
+  namespace: sparglim
+spec:
+  ports:
+    - name: connect-server
+      protocol: TCP
+      port: 15002
+      nodePort: 30052
+      targetPort: 15002
+    - name: spark-ui
+      protocol: TCP
+      port: 4040
+      nodePort: 30040
+      targetPort: 4040
+  selector:
+    app: sparglim-server
+  type: NodePort

dev/jupyter-sparglim-sc/k8s/sparglim-server/deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sparglim-server
+  namespace: sparglim
+  labels:
+    app: sparglim-server
+spec:
+  replicas: 1
+
+  selector:
+    matchLabels:
+      app: sparglim-server
+  template:
+    metadata:
+      labels:
+        app: sparglim-server
+    spec:
+      containers:
+        - name: sparglim-server
+          image: wh1isper/sparglim-server:dev
+          ports:
+            - containerPort: 15002
+          imagePullPolicy: Never
+          env:
+            - name: SPARGLIM_SERVER_MODE
+              value: "k8s"
+            - name: SPARGLIM_DRIVER_HOST
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: SPARGLIM_DRIVER_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name

examples/jupyter-sparglim-sc/k8s/headless-service.yaml → dev/jupyter-sparglim-sc/k8s/sparglim-server/headless-service.yaml

@@ -1,9 +1,9 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: sparglim-headless-service
+  name: sparglim-server-headless-service
   namespace: sparglim
 spec:
   clusterIP: None
   selector:
-    app: sparglim
+    app: sparglim-server

dev/sparglim-server/reload.sh → dev/scripts/reload.sh

@@ -1,3 +1,4 @@
 #!/usr/bin/env bash
 
 kubectl get pod -n sparglim |grep sparglim-server | awk '{print $(1)}' |xargs kubectl delete pod -n sparglim
+kubectl get pod -n sparglim |grep sparglim-app | awk '{print $(1)}' |xargs kubectl delete pod -n sparglim

examples/jupyter-sparglim-sc/k8s/deployment.yaml → examples/jupyter-sparglim-sc/k8s/jupyter-sparglim/deployment.yaml

@@ -24,7 +24,9 @@ spec:
           imagePullPolicy: Always
           env:
             - name: SPARGLIM_SQL_MODE
-              value: "k8s"
+              value: connect_client
+            - name: SPARGLIM_REMOTE
+              value: "sc://sparglim-server-headless-service:15002"
             - name: SPARGLIM_DRIVER_HOST
               valueFrom:
                 fieldRef:

examples/jupyter-sparglim-sc/k8s/jupyter-sparglim/lab-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sparglim-lab-service
+  namespace: sparglim
+spec:
+  ports:
+    - name: lab
+      protocol: TCP
+      port: 8888
+      nodePort: 30888
+      targetPort: 8888
+  selector:
+    app: sparglim-app
+  type: NodePort

examples/jupyter-sparglim-sc/k8s/sparglim-server/connect-server-service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sparglim-server-service
+  namespace: sparglim
+spec:
+  ports:
+    - name: connect-server
+      protocol: TCP
+      port: 15002
+      nodePort: 30052
+      targetPort: 15002
+    - name: spark-ui
+      protocol: TCP
+      port: 4040
+      nodePort: 30040
+      targetPort: 4040
+  selector:
+    app: sparglim-server
+  type: NodePort

examples/jupyter-sparglim-sc/k8s/sparglim-server/deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sparglim-server
+  namespace: sparglim
+  labels:
+    app: sparglim-server
+spec:
+  replicas: 1
+
+  selector:
+    matchLabels:
+      app: sparglim-server
+  template:
+    metadata:
+      labels:
+        app: sparglim-server
+    spec:
+      containers:
+        - name: sparglim-server
+          image: wh1isper/sparglim-server
+          ports:
+            - containerPort: 15002
+          imagePullPolicy: Always
+          env:
+            - name: SPARGLIM_SERVER_MODE
+              value: "k8s"
+            - name: SPARGLIM_DRIVER_HOST
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: SPARGLIM_DRIVER_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name

examples/jupyter-sparglim-sc/k8s/sparglim-server/headless-service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sparglim-server-headless-service
+  namespace: sparglim
+spec:
+  clusterIP: None
+  selector:
+    app: sparglim-server