404
+ +Page not found
+ + +'+ escapeHtml(title) + '
' + escapeHtml(summary) +'
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Welcome to the course: Introduction to Kebnekaise¶
+
+
+This material
+Here you will find the content of the workshop “Introduction to Kebnekaise”.
+You can download the markdown files for the presentation as well as the exercises from https://github.com/hpc2n/intro-course
+-
+
- Click the gren “Code” button
-
+
- Either copy the url for the repo under HTTPS and do
git clone https://github.com/hpc2n/intro-course.gitin a terminal window
+ - OR pick “Download zip” to get a zip file with the content. +
+ - Either copy the url for the repo under HTTPS and do
Some useful links:
+-
+
- Documentation about Linux at HPC2N: https://docs.hpc2n.umu.se/tutorials/linuxguide/ +
- Get started guide: https://docs.hpc2n.umu.se/tutorials/quickstart/ +
- Documentation pages at HPC2N: https://docs.hpc2n.umu.se/ +
+
+Prerequisites
+-
+
- Basic knowledge about Linux (if you need a refresher, you could take the course “Introduction to Linux” which runs immediately before this course. Info and registration here: https://www.hpc2n.umu.se/events/courses/2024/fall/intro-linux. +
- An account at SUPR and at HPC2N. You should have already been contacted about getting these if you did not have them already. +
+
+Content
+-
+
- This course aims to give a brief, but comprehensive introduction to Kebnekaise. +
- You will learn about +
- HPC2N, HPC, and Kebnekaise hardware +
- How to use our systems:
-
+
- Logging in +
- The File System +
- The Module System +
- Compiler Tool Chains +
- The Batch System +
+ - Simple examples (batch system) +
-
+
Application examples (batch system)
+
+ -
+
This course will consist of lectures and type-alongs, as well as a few exercises where you get to try out what you have just learned.
+
+
Instructors¶
+-
+
- Birgitte Brydsö, HPC2N +
- Pedro Ojeda May, HPC2N +
Preliminary schedule¶
+| Time | +Topic | +Activity | +
|---|---|---|
| 11:15 | +Welcome+Syllabus | ++ |
| 11:25 | +Introduction to Kebnekaise and HPC2N | +Lecture | +
| 11:45 | +Logging in | +Lecture+code along | +
| 11:55 | +The File System | +Lecture+code along | +
| 12:15 | +LUNCH BREAK | ++ |
| 13:15 | +The Module System | +Lecture+code along | +
| 13:35 | +Compiling, compiler tool chains | +Lecture+code along+exercise | +
| 13:50 | +The Batch System | +Lecture+code along | +
| 14:10 | +Simple Examples | +Lecture+exercises | +
| 14:45 | +COFFEE BREAK | ++ |
| 15:00 | +Application Examples | +Lecture+code along+exercises | +
| 16:40 | +Questions+Summary | ++ |
| 17:00 | +END OF COURSE | ++ |
+
+
+
+
+ Next »
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/info/GPU comparison.xlsx b/info/GPU comparison.xlsx
new file mode 100644
index 00000000..7a4f4b3e
Binary files /dev/null and b/info/GPU comparison.xlsx differ
diff --git a/intro/index.html b/intro/index.html
new file mode 100644
index 00000000..46e0d157
--- /dev/null
+++ b/intro/index.html
@@ -0,0 +1,745 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Introduction to HPC2N, Kebnekaise and HPC¶
+ + +
+
+
-
+
- Welcome page and syllabus: https://hpc2n.github.io/intro-linux/index.html
-
+
- Also link at the House symbol at the top of the page. +
+
HPC2N¶
+
+
+Note
+High Performance Computing Center North (HPC2N) is
+-
+
- a competence center for Scientific and Parallel Computing +
- part of National Academic Infrastructure for Supercomputing in Sweden (NAISS) +
HPC2N provides state-of-the-art resources and expertise:
+-
+
- Scalable and parallel HPC +
- Large-scale storage facilities (Project storage (Lustre), SweStore, Tape) +
- Grid and cloud computing (WLCG NT1, Swedish Science Cloud) +
- National Data Science Node in ”Epidemiology and Biology of Infections” (DDLS) +
- Software for e-Science applications +
- All levels of user support
-
+
- Primary, advanced, dedicated +
- Application Experts (AEs) +
+
+
+Primary objective
+To raise the national and local level of HPC competence and transfer HPC knowledge and technology to new users in academia and industry.
+HPC2N partners¶
+HPC2N is hosted by:
+
Partners:
+
+
+
+
HPC2N funding and collaborations¶
+Funded mainly by Umeå University, with contributions from the other HPC2N partners.
+Involved in several projects and collaborations:
+
+
+
+
+
+
HPC2N training and other services¶
+-
+
- User support (primary, advanced, dedicated)
-
+
- Research group meetings @ UmU +
- Also at the partner sites +
- Online “HPC2N fika” +
+ - User training and education program
-
+
- 0.5 – 5 days; ready-to-run exercises +
- Introduction to Linux, 16 September 2024 +
- Introduction to HPC2N and Kebnekaise, 16 September 2024 +
- Parallel programming and tools (OpenMP, MPI, debugging, perf. analyzers, Matlab, R, MD simulation, ML, GPU, …) +
- Basic Singularity, 16 October 2024 +
- Introduction to running R, Python, Julia, and Matlab in HPC, 22-25 October 2024 +
- Introduction to Git, 25-29 November 2024 +
- Using Python in an HPC environment, 5-6 December 2024 +
- Updated list: https://www.hpc2n.umu.se/events/courses +
+ - Workshops and seminars +
- NGSSC / SeSE & university courses +
HPC2N personnel¶
+Management:
+-
+
- Paolo Bientinesi, director +
- Björn Torkelsson, deputy director +
- Lena Hellman, administrator +
Application experts:
+-
+
- Jerry Eriksson +
- Pedro Ojeda May +
- Birgitte Brydsö +
- Åke Sandgren +
Others:
+-
+
- Mikael Rännar (WLCG coord) +
- Research Engineers under DDLS, HPC2N/SciLifeLab
-
+
- Paul Dulaud, System Developer, IT +
- Abdullah Aziz, Data Engineer +
- Data Steward +
+
System and support:
+-
+
- Erik Andersson +
- Birgitte Brydsö +
- Niklas Edmundsson (Tape coord) +
- My Karlsson +
- Roger Oscarsson +
- Åke Sandgren +
- Mattias Wadenstein (NeIC, Tier1) +
- Lars Viklund +
HPC2N application experts¶
+-
+
- HPC2N provides advanced and dedicated support in the form of Application Experts (AEs):
-
+
- Jerry Eriksson: Profiling, Machine learning (DNN), MPI, OpenMP, OpenACC +
- Pedro Ojeda May: Molecular dynamics, Profiling, QM/MM, NAMD, Amber, Gromacs, GAUSSIAN, R, Python +
- Åke Sandgren: General high level programming assistance, VASP, Gromacs, Amber +
- Birgitte Brydsö: General HPC, R, Python +
+ - Contact through regular support +
HPC2N users by discipline¶
+-
+
- Users from several scientific disciplines:
-
+
- Biosciences and medicine +
- Chemistry +
- Computing science +
- Engineering +
- Materials science +
- Mathematics and statistics +
- Physics including space physics +
- ML, DL, and other AI +
+
HPC2N users by discipline, largest users¶
+-
+
- Users from several scientific disciplines:
-
+
- Biosciences and medicine +
- Chemistry +
- Computing science +
- Engineering +
- Materials science +
- Mathematics and statistics +
- Physics including space physics +
- Machine learning and artificial intelligence (several new projects) +
+
HPC2N users by software¶
+
Kebnekaise¶
+The current supercomputer at HPC2N. It is a very heterogeneous system.
+-
+
- Named after a massif (contains some of Sweden’s highest mountain peaks) +
- Kebnekaise was
-
+
- delivered by Lenovo and +
- installed during the summer 2016 +
- Opened up for general availability on November 7, 2016 +
- In 2018, Kebnekaise was extended with
-
+
- 52 Intel Xeon Gold 6132 (Skylake) nodes, as well as +
- 10 NVidian V100 (Volta) GPU nodes +
+ - In 2023, Kebnekaise was extended with
-
+
- 2 dual NVIDIA A100 GPU nodes +
- one many-core AMD Zen3 CPU node +
+
+
Kebnekaise will be continuosly upgraded, as old hardware gets retired.
+-
+
- In 2024 Kebnekaise was extended with
-
+
- 2 Dual socket GPU-nodes: Lenovo ThinkSystem SR675 V3
-
+
- 2 x AMD EPYC 9454 48C 290W 2.75GHz Processor +
- 768GB [24x 32GB TruDDR5 4800MHz RDIMM-A] +
- 1 x 3.84TB Read Intensive NVMe PCIe 4.0 x4 HS SSD +
- 1 x NVIDIA H100 SXM5 700W 80G HBM3 GPU Board +
+ - 10 dual-socket GPU-nodes: ThinkSystem SR665 V3
-
+
- 2 x AMD EPYC 9254 24C 200W 2.9GHz Processor +
- 384GB [24x 16GB TruDDR5 4800MHz RDIMM-A] +
- 1 x 1.92TB Read Intensive NVMe PCIe 5.0 x4 HS SSD +
- 2 x NVIDIA L40S 48GB PCIe Gen4 Passive GPU +
+ - 8 dual-socket CPU only: ThinkSystem SR645 V3
-
+
- 2 x AMD EPYC 9754 128C 360W 2.25GHz Processor +
- 768GB [24x 32GB TruDDR5 4800MHz RDIMM-A] +
- 1 x 1 3.84TB Read Intensive NVMe PCIe 4.0 x4 HS SSD +
+
+ - 2 Dual socket GPU-nodes: Lenovo ThinkSystem SR675 V3
Current hardware in Kebnekaise¶
+Kebnekaise have CPU-only, GPU enabled and large memory nodes.
+The CPU-only nodes are:
+-
+
- 2 x 14 core Intel broadwell
-
+
- 4460 MB memory / core +
- 48 nodes +
- Total of 41.6 TFlops/s +
+ - 2 x 14 core Intel skylake
-
+
- 6785 MB memory / core +
- 52 nodes +
- Total of 87 TFlops/s +
+ - 2 x 64 core AMD zen3
-
+
- 8020 MB / core +
- 1 node +
- Total of 11 TFlops/s +
+ - 2 x 128 core AMD zen4
-
+
- 2516 MB / core +
- 8 nodes +
- Total of 216 TFlops/s +
+
The GPU enabled nodes are:
+-
+
- 2 x 14 core Intel broadwell
-
+
- 9000 MB memory / core +
- 2 x Nvidia A40 +
- 4 nodes +
- Total of 83 TFlops/s +
+ - 2 x 14 core Intel skylake
-
+
- 6785 MB memory / core +
- 2 x Nvidia V100 +
- 10 nodes +
- Total of 75 TFlops/s +
+ - 2 x 24 core AMD zen3
-
+
- 10600 MB / core +
- 2 x Nvidia A100 +
- 2 nodes +
+ - 2 x 24 core AMD zen3
-
+
- 10600 MB / core +
- 2 x AMD MI100 +
- 1 node +
+ - 2 x 24 core AMD zen4
-
+
- 6630 MB / core +
- 2 x Nvidia A6000 +
- 1 node +
+ - 2 x 24 core AMD zen4
-
+
- 6630 MB / core +
- 2 x Nvidia L40s +
- 10 nodes +
+ - 2 x 48 core AMD zen4
-
+
- 6630 MB / core +
- 4 x Nvidia H100 SXM5 +
- 2 nodes +
+
The large memory nodes are:
+-
+
- 4 x 18 core Intel broadwell
-
+
- 41666 MB memory / core +
- 8 nodes +
- Total of 13.6 TFlops/s for all these nodes +
+
Kebnekaise - HPC2N storage¶
+Basically four types of storage are available at HPC2N:
+-
+
- Home directory
-
+
/home/X/Xyz,$HOME,~
+- 25 GB, user owned +
+ - Project storage
-
+
/proj/nobackup/abc
+- Shared among project members +
+ - Local scratch space
-
+
$SNIC_TMP
+- SSD (170GB), per job, per node, “volatile” +
+ - Tape Storage
-
+
- Backup +
- Long term storage +
+
Also
+-
+
- SweStore — disk based (dCache)
-
+
- Research Data Storage Infrastructure, for active research data and operated by NAISS, WLCG +
+
Kebnekaise - projects¶
+
+
+Compute projects
+To use Kebnekaise, you must be a member of a compute project.
+-
+
- A compute project has a certain number of core hours allocated for it per month +
- A regular CPU core cost 1 core hour per hour, other resources (e.g., GPUs) cost more +
- Not a hard limit but projects that go over the allocation get lower priority +
A compute project contains a certain amount of storage. If more storage is required, you must be a member of a storage project.
+
+
+Note
+As Kebnekaise is a local cluster, you need to be affiliated with UmU, IRF, SLU, Miun, or LTU to use it.
+Projects are applied for through SUPR (https://supr.naiss.se).
+I will cover more details in a later section, where we go more into detail about HPC2N and Kebnekaise.
+HPC¶
+
+
+What is HPC?
+High Performance Computing (definition)
+“High Performance Computing most generally refers to the practice of aggregating computing power in a way that delivers much higher performance than one could get out of a typical desktop computer or workstation in order to solve large problems in science, engineering, or business.”
+ +High Performance Computing - opening the definition¶
+Aggregating computing power¶
+-
+
- 147 nodes totalling 6808 cores and ??? CUDA cores
-
+
- Compared to 4 cores in a modern laptop +
+
Higher performance¶
+-
+
- More than 527,000,000,000,000 arithmetical operations per second (527 trillion (billion)
-
+
- Compared to 200,000,000,000 Flops in a modern laptop (200 billion (milliard) +
+
Solve large problems¶
+-
+
- When does a problem become large enough for HPC? +
- Are there other reasons for using HPC resources? (Memory, software, support, etc.) +
High Performance Computing - large problems¶
+A problem can be large for two main reasons:
+-
+
- Execution time: The time required to form a solution to the problem is very long +
- Memory / storage use: The solution of the problem requires a lot of memory and/or storage +
The former can be remedied by increasing the performance
+-
+
- More cores, more nodes, GPUs, … +
The latter by adding more memory / storage
+-
+
- More memory per node (including large memory nodes), more nodes, …
-
+
- Kebnekaise: 128GB - 192GB, 384GB, 512GB, 768GB, 3TB +
+ - Large storage solutions, … +
High Performance Computing - what counts as HPC¶
+
High Performance Computing - other reasons¶
+-
+
- Specialized (expensive) hardware
-
+
- GPUs, including those optimized for AI
-
+
- Kebnekaise has V100, A100, A40, MI100, A6000, L40S, H100 +
+ - High-end CPUs (AVX-512 etc) and ECC memory +
+ - GPUs, including those optimized for AI
- Software
-
+
- HPC2N holds licenses for several softwares +
- Software is pre-configured and ready-to-use +
+ - Support and documentation +
High Performance Computing - memory models¶
+Two memory models are relevant for HPC:
+-
+
- Shared memory: Single memory space for all data.
+
-
+
- Everyone can access the same data +
- Straightforward to use +
+ - Distributed memory: Multiple distinct memory spaces.
+
-
+
- Everyone has direct access only to the local data +
- Requires communication +
+
\frametitle{High Performance Computing (memory models)} +\begin{center} + \includegraphics[width=0.6\textwidth]{figures/memory.pdf} +\end{center} +}
+\frame{ +\frametitle{High Performance Computing (programming models)} +\begin{itemize} + \item The programming model changes when we aim for extra performance and/or memory: + \begin{enumerate} + \pause \item {\color{cyan} Single-core}: Matlab, Python, C, Fortran, \dots + \begin{itemize} + \item Single stream of operations + \end{itemize} + \pause \item {\color{cyan} Multi-core}: Vectorized Matlab, pthreads, \emph{OpenMP} + \begin{itemize} + \item \emph{Multiple streams} of operations + \pause \item \emph{Work distribution}, \emph{coordination} (synchronization, etc), \dots + \end{itemize} + \pause \item {\color{cyan} Distributed memory}: \emph{MPI}, \dots + \begin{itemize} + \item Multiple streams of operations + \item Work distribution, coordination (synchronization, etc), \dots + \pause \item \emph{Data distribution and communication} + \end{itemize} + \end{enumerate} + \pause \item {\color{cyan} GPUs}: \emph{CUDA}, OpenCL, OpenACC, OpenMP, \dots + \begin{itemize} + \pause \item \emph{Many lightweight} streams of operations + \pause \item Work distribution, coordination (synchronization, etc), \dots + \pause \item \emph{Data distribution across memory spaces and movement} + \end{itemize} +\end{itemize} +}
+\frame{ +\frametitle{High Performance Computing (software)} +\begin{itemize} + \item Complexity grows when we aim for extra performance and/or memory/storage: + \begin{enumerate} + \pause \item {\color{cyan} Single-core}: LAPACK, \dots + \begin{itemize} + \item Load correct toolchain etc + \end{itemize} + \pause \item {\color{cyan} Multi-core}: LAPACK + parallel BLAS, \dots + \begin{itemize} + \item Load correct toolchain etc + \pause \item \emph{Allocate} correct number of cores, \emph{configure} software to use correct number of cores, \dots + \end{itemize} + \pause \item {\color{cyan} Distributed memory}: ScaLAPACK, \dots + \begin{itemize} + \item Load correct toolchain etc + \pause \item Allocate correct number of \emph{nodes and cores}, configure software to use correct number of \emph{nodes and cores}, \dots + \pause \item Data distribution, storage, \dots + \end{itemize} + \end{enumerate} + \pause \item {\color{cyan} GPUs}: MAGMA, TensorFlow, \dots + \begin{itemize} + \item Load correct toolchain etc + \item Allocate correct number of \emph{cores and GPUs}, configure software to use correct number of \emph{cores and GPUs}, \dots + \end{itemize} +\end{itemize} +}
+\frame{ +\frametitle{End (questions?)} +\centering \Huge Questions? +}
+\end{document}
+
+
+Learning outcomes
+-
+
- What is Linux? +
- Learn about the command line interface (CLI) +
- Navigate the file system - ls, mkdir, cd, rm, cp, mv, redirection, pipes, etc. +
- Data handling - compressing, archiving, file transfer, patterns, etc. +
- Hints and tricks of Linux +
Why Linux¶
+-
+
- Most HPC centers supercomputers run some flavour of Linux. +
- It is stable and reliable +
- Customizable +
- Runs on any hardware +
- Strong community +
- Many flavours are open source and free +
- Lots of applications +
- Lightweight +
What is Linux¶
+Linux is a family of open-source Unix-like operating systems based on the Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds.
+An operating system is the software that sits underneath all of the other software on a computer, managing the computer’s hardware (CPU, GPU, memory, storage…) and taking care of the connections between your other software and the hardware.
+Linux is typically packaged as a Linux distribution, which includes the kernel and supporting system software and libraries, many of which are provided by the GNU Project.
+
+
+Distributions (distros)
+There are many Linux distribuitions, including Ubuntu, Debian, Fedora, Gentoo, and many others. Many distributions are free and open source, but there are also commercial distributions, like Red Hat Enterprise and SUSE.
+Desktop Linux distributions include a desktop environment, like GNOME, MATE, KDE Plasma, Xfce, Unity, or many others. A window manager together with applications written using a widget toolkit are generally responsible for most of what the user sees.
+In addition, a windowing system of some sort (X11, Wayland) interfaces directly with the underlying operating system and libraries, providing support for graphical hardware, pointing devices, and keyboards. The window manager generally runs on top of this windowing system.
+While the use of Linux on desktop is only 2-3%, the vast majority of web servers (>96%), most mobile devices (Android is based on the Linux kernel), and all supercomputers on the Top500 list run Linux.
+More information¶
+There is much more information about Linux on Wikipedia.
+Some pages with guides and/or cheat sheets:
+ + +
+
+
+
+ « Previous
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/js/html5shiv.min.js b/js/html5shiv.min.js
new file mode 100644
index 00000000..1a01c94b
--- /dev/null
+++ b/js/html5shiv.min.js
@@ -0,0 +1,4 @@
+/**
+* @preserve HTML5 Shiv 3.7.3 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed
+*/
+!function(a,b){function c(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x",d.insertBefore(c.lastChild,d.firstChild)}function d(){var a=t.elements;return"string"==typeof a?a.split(" "):a}function e(a,b){var c=t.elements;"string"!=typeof c&&(c=c.join(" ")),"string"!=typeof a&&(a=a.join(" ")),t.elements=c+" "+a,j(b)}function f(a){var b=s[a[q]];return b||(b={},r++,a[q]=r,s[r]=b),b}function g(a,c,d){if(c||(c=b),l)return c.createElement(a);d||(d=f(c));var e;return e=d.cache[a]?d.cache[a].cloneNode():p.test(a)?(d.cache[a]=d.createElem(a)).cloneNode():d.createElem(a),!e.canHaveChildren||o.test(a)||e.tagUrn?e:d.frag.appendChild(e)}function h(a,c){if(a||(a=b),l)return a.createDocumentFragment();c=c||f(a);for(var e=c.frag.cloneNode(),g=0,h=d(),i=h.length;i>g;g++)e.createElement(h[g]);return e}function i(a,b){b.cache||(b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag()),a.createElement=function(c){return t.shivMethods?g(c,a,b):b.createElem(c)},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+d().join().replace(/[\w\-:]+/g,function(a){return b.createElem(a),b.frag.createElement(a),'c("'+a+'")'})+");return n}")(t,b.frag)}function j(a){a||(a=b);var d=f(a);return!t.shivCSS||k||d.hasCSS||(d.hasCSS=!!c(a,"article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}mark{background:#FF0;color:#000}template{display:none}")),l||i(a,d),a}var k,l,m="3.7.3",n=a.html5||{},o=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,p=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,q="_html5shiv",r=0,s={};!function(){try{var a=b.createElement("a");a.innerHTML="
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Search Results
+ + + +
+ Searching...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/search/lunr.js b/search/lunr.js
new file mode 100644
index 00000000..aca0a167
--- /dev/null
+++ b/search/lunr.js
@@ -0,0 +1,3475 @@
+/**
+ * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.9
+ * Copyright (C) 2020 Oliver Nightingale
+ * @license MIT
+ */
+
+;(function(){
+
+/**
+ * A convenience function for configuring and constructing
+ * a new lunr Index.
+ *
+ * A lunr.Builder instance is created and the pipeline setup
+ * with a trimmer, stop word filter and stemmer.
+ *
+ * This builder object is yielded to the configuration function
+ * that is passed as a parameter, allowing the list of fields
+ * and other builder parameters to be customised.
+ *
+ * All documents _must_ be added within the passed config function.
+ *
+ * @example
+ * var idx = lunr(function () {
+ * this.field('title')
+ * this.field('body')
+ * this.ref('id')
+ *
+ * documents.forEach(function (doc) {
+ * this.add(doc)
+ * }, this)
+ * })
+ *
+ * @see {@link lunr.Builder}
+ * @see {@link lunr.Pipeline}
+ * @see {@link lunr.trimmer}
+ * @see {@link lunr.stopWordFilter}
+ * @see {@link lunr.stemmer}
+ * @namespace {function} lunr
+ */
+var lunr = function (config) {
+ var builder = new lunr.Builder
+
+ builder.pipeline.add(
+ lunr.trimmer,
+ lunr.stopWordFilter,
+ lunr.stemmer
+ )
+
+ builder.searchPipeline.add(
+ lunr.stemmer
+ )
+
+ config.call(builder, builder)
+ return builder.build()
+}
+
+lunr.version = "2.3.9"
+/*!
+ * lunr.utils
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * A namespace containing utils for the rest of the lunr library
+ * @namespace lunr.utils
+ */
+lunr.utils = {}
+
+/**
+ * Print a warning message to the console.
+ *
+ * @param {String} message The message to be printed.
+ * @memberOf lunr.utils
+ * @function
+ */
+lunr.utils.warn = (function (global) {
+ /* eslint-disable no-console */
+ return function (message) {
+ if (global.console && console.warn) {
+ console.warn(message)
+ }
+ }
+ /* eslint-enable no-console */
+})(this)
+
+/**
+ * Convert an object to a string.
+ *
+ * In the case of `null` and `undefined` the function returns
+ * the empty string, in all other cases the result of calling
+ * `toString` on the passed object is returned.
+ *
+ * @param {Any} obj The object to convert to a string.
+ * @return {String} string representation of the passed object.
+ * @memberOf lunr.utils
+ */
+lunr.utils.asString = function (obj) {
+ if (obj === void 0 || obj === null) {
+ return ""
+ } else {
+ return obj.toString()
+ }
+}
+
+/**
+ * Clones an object.
+ *
+ * Will create a copy of an existing object such that any mutations
+ * on the copy cannot affect the original.
+ *
+ * Only shallow objects are supported, passing a nested object to this
+ * function will cause a TypeError.
+ *
+ * Objects with primitives, and arrays of primitives are supported.
+ *
+ * @param {Object} obj The object to clone.
+ * @return {Object} a clone of the passed object.
+ * @throws {TypeError} when a nested object is passed.
+ * @memberOf Utils
+ */
+lunr.utils.clone = function (obj) {
+ if (obj === null || obj === undefined) {
+ return obj
+ }
+
+ var clone = Object.create(null),
+ keys = Object.keys(obj)
+
+ for (var i = 0; i < keys.length; i++) {
+ var key = keys[i],
+ val = obj[key]
+
+ if (Array.isArray(val)) {
+ clone[key] = val.slice()
+ continue
+ }
+
+ if (typeof val === 'string' ||
+ typeof val === 'number' ||
+ typeof val === 'boolean') {
+ clone[key] = val
+ continue
+ }
+
+ throw new TypeError("clone is not deep and does not support nested objects")
+ }
+
+ return clone
+}
+lunr.FieldRef = function (docRef, fieldName, stringValue) {
+ this.docRef = docRef
+ this.fieldName = fieldName
+ this._stringValue = stringValue
+}
+
+lunr.FieldRef.joiner = "/"
+
+lunr.FieldRef.fromString = function (s) {
+ var n = s.indexOf(lunr.FieldRef.joiner)
+
+ if (n === -1) {
+ throw "malformed field ref string"
+ }
+
+ var fieldRef = s.slice(0, n),
+ docRef = s.slice(n + 1)
+
+ return new lunr.FieldRef (docRef, fieldRef, s)
+}
+
+lunr.FieldRef.prototype.toString = function () {
+ if (this._stringValue == undefined) {
+ this._stringValue = this.fieldName + lunr.FieldRef.joiner + this.docRef
+ }
+
+ return this._stringValue
+}
+/*!
+ * lunr.Set
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * A lunr set.
+ *
+ * @constructor
+ */
+lunr.Set = function (elements) {
+ this.elements = Object.create(null)
+
+ if (elements) {
+ this.length = elements.length
+
+ for (var i = 0; i < this.length; i++) {
+ this.elements[elements[i]] = true
+ }
+ } else {
+ this.length = 0
+ }
+}
+
+/**
+ * A complete set that contains all elements.
+ *
+ * @static
+ * @readonly
+ * @type {lunr.Set}
+ */
+lunr.Set.complete = {
+ intersect: function (other) {
+ return other
+ },
+
+ union: function () {
+ return this
+ },
+
+ contains: function () {
+ return true
+ }
+}
+
+/**
+ * An empty set that contains no elements.
+ *
+ * @static
+ * @readonly
+ * @type {lunr.Set}
+ */
+lunr.Set.empty = {
+ intersect: function () {
+ return this
+ },
+
+ union: function (other) {
+ return other
+ },
+
+ contains: function () {
+ return false
+ }
+}
+
+/**
+ * Returns true if this set contains the specified object.
+ *
+ * @param {object} object - Object whose presence in this set is to be tested.
+ * @returns {boolean} - True if this set contains the specified object.
+ */
+lunr.Set.prototype.contains = function (object) {
+ return !!this.elements[object]
+}
+
+/**
+ * Returns a new set containing only the elements that are present in both
+ * this set and the specified set.
+ *
+ * @param {lunr.Set} other - set to intersect with this set.
+ * @returns {lunr.Set} a new set that is the intersection of this and the specified set.
+ */
+
+lunr.Set.prototype.intersect = function (other) {
+ var a, b, elements, intersection = []
+
+ if (other === lunr.Set.complete) {
+ return this
+ }
+
+ if (other === lunr.Set.empty) {
+ return other
+ }
+
+ if (this.length < other.length) {
+ a = this
+ b = other
+ } else {
+ a = other
+ b = this
+ }
+
+ elements = Object.keys(a.elements)
+
+ for (var i = 0; i < elements.length; i++) {
+ var element = elements[i]
+ if (element in b.elements) {
+ intersection.push(element)
+ }
+ }
+
+ return new lunr.Set (intersection)
+}
+
+/**
+ * Returns a new set combining the elements of this and the specified set.
+ *
+ * @param {lunr.Set} other - set to union with this set.
+ * @return {lunr.Set} a new set that is the union of this and the specified set.
+ */
+
+lunr.Set.prototype.union = function (other) {
+ if (other === lunr.Set.complete) {
+ return lunr.Set.complete
+ }
+
+ if (other === lunr.Set.empty) {
+ return this
+ }
+
+ return new lunr.Set(Object.keys(this.elements).concat(Object.keys(other.elements)))
+}
+/**
+ * A function to calculate the inverse document frequency for
+ * a posting. This is shared between the builder and the index
+ *
+ * @private
+ * @param {object} posting - The posting for a given term
+ * @param {number} documentCount - The total number of documents.
+ */
+lunr.idf = function (posting, documentCount) {
+ var documentsWithTerm = 0
+
+ for (var fieldName in posting) {
+ if (fieldName == '_index') continue // Ignore the term index, its not a field
+ documentsWithTerm += Object.keys(posting[fieldName]).length
+ }
+
+ var x = (documentCount - documentsWithTerm + 0.5) / (documentsWithTerm + 0.5)
+
+ return Math.log(1 + Math.abs(x))
+}
+
+/**
+ * A token wraps a string representation of a token
+ * as it is passed through the text processing pipeline.
+ *
+ * @constructor
+ * @param {string} [str=''] - The string token being wrapped.
+ * @param {object} [metadata={}] - Metadata associated with this token.
+ */
+lunr.Token = function (str, metadata) {
+ this.str = str || ""
+ this.metadata = metadata || {}
+}
+
+/**
+ * Returns the token string that is being wrapped by this object.
+ *
+ * @returns {string}
+ */
+lunr.Token.prototype.toString = function () {
+ return this.str
+}
+
+/**
+ * A token update function is used when updating or optionally
+ * when cloning a token.
+ *
+ * @callback lunr.Token~updateFunction
+ * @param {string} str - The string representation of the token.
+ * @param {Object} metadata - All metadata associated with this token.
+ */
+
+/**
+ * Applies the given function to the wrapped string token.
+ *
+ * @example
+ * token.update(function (str, metadata) {
+ * return str.toUpperCase()
+ * })
+ *
+ * @param {lunr.Token~updateFunction} fn - A function to apply to the token string.
+ * @returns {lunr.Token}
+ */
+lunr.Token.prototype.update = function (fn) {
+ this.str = fn(this.str, this.metadata)
+ return this
+}
+
+/**
+ * Creates a clone of this token. Optionally a function can be
+ * applied to the cloned token.
+ *
+ * @param {lunr.Token~updateFunction} [fn] - An optional function to apply to the cloned token.
+ * @returns {lunr.Token}
+ */
+lunr.Token.prototype.clone = function (fn) {
+ fn = fn || function (s) { return s }
+ return new lunr.Token (fn(this.str, this.metadata), this.metadata)
+}
+/*!
+ * lunr.tokenizer
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * A function for splitting a string into tokens ready to be inserted into
+ * the search index. Uses `lunr.tokenizer.separator` to split strings, change
+ * the value of this property to change how strings are split into tokens.
+ *
+ * This tokenizer will convert its parameter to a string by calling `toString` and
+ * then will split this string on the character in `lunr.tokenizer.separator`.
+ * Arrays will have their elements converted to strings and wrapped in a lunr.Token.
+ *
+ * Optional metadata can be passed to the tokenizer, this metadata will be cloned and
+ * added as metadata to every token that is created from the object to be tokenized.
+ *
+ * @static
+ * @param {?(string|object|object[])} obj - The object to convert into tokens
+ * @param {?object} metadata - Optional metadata to associate with every token
+ * @returns {lunr.Token[]}
+ * @see {@link lunr.Pipeline}
+ */
+lunr.tokenizer = function (obj, metadata) {
+ if (obj == null || obj == undefined) {
+ return []
+ }
+
+ if (Array.isArray(obj)) {
+ return obj.map(function (t) {
+ return new lunr.Token(
+ lunr.utils.asString(t).toLowerCase(),
+ lunr.utils.clone(metadata)
+ )
+ })
+ }
+
+ var str = obj.toString().toLowerCase(),
+ len = str.length,
+ tokens = []
+
+ for (var sliceEnd = 0, sliceStart = 0; sliceEnd <= len; sliceEnd++) {
+ var char = str.charAt(sliceEnd),
+ sliceLength = sliceEnd - sliceStart
+
+ if ((char.match(lunr.tokenizer.separator) || sliceEnd == len)) {
+
+ if (sliceLength > 0) {
+ var tokenMetadata = lunr.utils.clone(metadata) || {}
+ tokenMetadata["position"] = [sliceStart, sliceLength]
+ tokenMetadata["index"] = tokens.length
+
+ tokens.push(
+ new lunr.Token (
+ str.slice(sliceStart, sliceEnd),
+ tokenMetadata
+ )
+ )
+ }
+
+ sliceStart = sliceEnd + 1
+ }
+
+ }
+
+ return tokens
+}
+
+/**
+ * The separator used to split a string into tokens. Override this property to change the behaviour of
+ * `lunr.tokenizer` behaviour when tokenizing strings. By default this splits on whitespace and hyphens.
+ *
+ * @static
+ * @see lunr.tokenizer
+ */
+lunr.tokenizer.separator = /[\s\-]+/
+/*!
+ * lunr.Pipeline
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * lunr.Pipelines maintain an ordered list of functions to be applied to all
+ * tokens in documents entering the search index and queries being ran against
+ * the index.
+ *
+ * An instance of lunr.Index created with the lunr shortcut will contain a
+ * pipeline with a stop word filter and an English language stemmer. Extra
+ * functions can be added before or after either of these functions or these
+ * default functions can be removed.
+ *
+ * When run the pipeline will call each function in turn, passing a token, the
+ * index of that token in the original list of all tokens and finally a list of
+ * all the original tokens.
+ *
+ * The output of functions in the pipeline will be passed to the next function
+ * in the pipeline. To exclude a token from entering the index the function
+ * should return undefined, the rest of the pipeline will not be called with
+ * this token.
+ *
+ * For serialisation of pipelines to work, all functions used in an instance of
+ * a pipeline should be registered with lunr.Pipeline. Registered functions can
+ * then be loaded. If trying to load a serialised pipeline that uses functions
+ * that are not registered an error will be thrown.
+ *
+ * If not planning on serialising the pipeline then registering pipeline functions
+ * is not necessary.
+ *
+ * @constructor
+ */
+lunr.Pipeline = function () {
+ this._stack = []
+}
+
+lunr.Pipeline.registeredFunctions = Object.create(null)
+
+/**
+ * A pipeline function maps lunr.Token to lunr.Token. A lunr.Token contains the token
+ * string as well as all known metadata. A pipeline function can mutate the token string
+ * or mutate (or add) metadata for a given token.
+ *
+ * A pipeline function can indicate that the passed token should be discarded by returning
+ * null, undefined or an empty string. This token will not be passed to any downstream pipeline
+ * functions and will not be added to the index.
+ *
+ * Multiple tokens can be returned by returning an array of tokens. Each token will be passed
+ * to any downstream pipeline functions and all will returned tokens will be added to the index.
+ *
+ * Any number of pipeline functions may be chained together using a lunr.Pipeline.
+ *
+ * @interface lunr.PipelineFunction
+ * @param {lunr.Token} token - A token from the document being processed.
+ * @param {number} i - The index of this token in the complete list of tokens for this document/field.
+ * @param {lunr.Token[]} tokens - All tokens for this document/field.
+ * @returns {(?lunr.Token|lunr.Token[])}
+ */
+
+/**
+ * Register a function with the pipeline.
+ *
+ * Functions that are used in the pipeline should be registered if the pipeline
+ * needs to be serialised, or a serialised pipeline needs to be loaded.
+ *
+ * Registering a function does not add it to a pipeline, functions must still be
+ * added to instances of the pipeline for them to be used when running a pipeline.
+ *
+ * @param {lunr.PipelineFunction} fn - The function to check for.
+ * @param {String} label - The label to register this function with
+ */
+lunr.Pipeline.registerFunction = function (fn, label) {
+ if (label in this.registeredFunctions) {
+ lunr.utils.warn('Overwriting existing registered function: ' + label)
+ }
+
+ fn.label = label
+ lunr.Pipeline.registeredFunctions[fn.label] = fn
+}
+
+/**
+ * Warns if the function is not registered as a Pipeline function.
+ *
+ * @param {lunr.PipelineFunction} fn - The function to check for.
+ * @private
+ */
+lunr.Pipeline.warnIfFunctionNotRegistered = function (fn) {
+ var isRegistered = fn.label && (fn.label in this.registeredFunctions)
+
+ if (!isRegistered) {
+ lunr.utils.warn('Function is not registered with pipeline. This may cause problems when serialising the index.\n', fn)
+ }
+}
+
+/**
+ * Loads a previously serialised pipeline.
+ *
+ * All functions to be loaded must already be registered with lunr.Pipeline.
+ * If any function from the serialised data has not been registered then an
+ * error will be thrown.
+ *
+ * @param {Object} serialised - The serialised pipeline to load.
+ * @returns {lunr.Pipeline}
+ */
+lunr.Pipeline.load = function (serialised) {
+ var pipeline = new lunr.Pipeline
+
+ serialised.forEach(function (fnName) {
+ var fn = lunr.Pipeline.registeredFunctions[fnName]
+
+ if (fn) {
+ pipeline.add(fn)
+ } else {
+ throw new Error('Cannot load unregistered function: ' + fnName)
+ }
+ })
+
+ return pipeline
+}
+
+/**
+ * Adds new functions to the end of the pipeline.
+ *
+ * Logs a warning if the function has not been registered.
+ *
+ * @param {lunr.PipelineFunction[]} functions - Any number of functions to add to the pipeline.
+ */
+lunr.Pipeline.prototype.add = function () {
+ var fns = Array.prototype.slice.call(arguments)
+
+ fns.forEach(function (fn) {
+ lunr.Pipeline.warnIfFunctionNotRegistered(fn)
+ this._stack.push(fn)
+ }, this)
+}
+
+/**
+ * Adds a single function after a function that already exists in the
+ * pipeline.
+ *
+ * Logs a warning if the function has not been registered.
+ *
+ * @param {lunr.PipelineFunction} existingFn - A function that already exists in the pipeline.
+ * @param {lunr.PipelineFunction} newFn - The new function to add to the pipeline.
+ */
+lunr.Pipeline.prototype.after = function (existingFn, newFn) {
+ lunr.Pipeline.warnIfFunctionNotRegistered(newFn)
+
+ var pos = this._stack.indexOf(existingFn)
+ if (pos == -1) {
+ throw new Error('Cannot find existingFn')
+ }
+
+ pos = pos + 1
+ this._stack.splice(pos, 0, newFn)
+}
+
+/**
+ * Adds a single function before a function that already exists in the
+ * pipeline.
+ *
+ * Logs a warning if the function has not been registered.
+ *
+ * @param {lunr.PipelineFunction} existingFn - A function that already exists in the pipeline.
+ * @param {lunr.PipelineFunction} newFn - The new function to add to the pipeline.
+ */
+lunr.Pipeline.prototype.before = function (existingFn, newFn) {
+ lunr.Pipeline.warnIfFunctionNotRegistered(newFn)
+
+ var pos = this._stack.indexOf(existingFn)
+ if (pos == -1) {
+ throw new Error('Cannot find existingFn')
+ }
+
+ this._stack.splice(pos, 0, newFn)
+}
+
+/**
+ * Removes a function from the pipeline.
+ *
+ * @param {lunr.PipelineFunction} fn The function to remove from the pipeline.
+ */
+lunr.Pipeline.prototype.remove = function (fn) {
+ var pos = this._stack.indexOf(fn)
+ if (pos == -1) {
+ return
+ }
+
+ this._stack.splice(pos, 1)
+}
+
+/**
+ * Runs the current list of functions that make up the pipeline against the
+ * passed tokens.
+ *
+ * @param {Array} tokens The tokens to run through the pipeline.
+ * @returns {Array}
+ */
+lunr.Pipeline.prototype.run = function (tokens) {
+ var stackLength = this._stack.length
+
+ for (var i = 0; i < stackLength; i++) {
+ var fn = this._stack[i]
+ var memo = []
+
+ for (var j = 0; j < tokens.length; j++) {
+ var result = fn(tokens[j], j, tokens)
+
+ if (result === null || result === void 0 || result === '') continue
+
+ if (Array.isArray(result)) {
+ for (var k = 0; k < result.length; k++) {
+ memo.push(result[k])
+ }
+ } else {
+ memo.push(result)
+ }
+ }
+
+ tokens = memo
+ }
+
+ return tokens
+}
+
+/**
+ * Convenience method for passing a string through a pipeline and getting
+ * strings out. This method takes care of wrapping the passed string in a
+ * token and mapping the resulting tokens back to strings.
+ *
+ * @param {string} str - The string to pass through the pipeline.
+ * @param {?object} metadata - Optional metadata to associate with the token
+ * passed to the pipeline.
+ * @returns {string[]}
+ */
+lunr.Pipeline.prototype.runString = function (str, metadata) {
+ var token = new lunr.Token (str, metadata)
+
+ return this.run([token]).map(function (t) {
+ return t.toString()
+ })
+}
+
+/**
+ * Resets the pipeline by removing any existing processors.
+ *
+ */
+lunr.Pipeline.prototype.reset = function () {
+ this._stack = []
+}
+
+/**
+ * Returns a representation of the pipeline ready for serialisation.
+ *
+ * Logs a warning if the function has not been registered.
+ *
+ * @returns {Array}
+ */
+lunr.Pipeline.prototype.toJSON = function () {
+ return this._stack.map(function (fn) {
+ lunr.Pipeline.warnIfFunctionNotRegistered(fn)
+
+ return fn.label
+ })
+}
+/*!
+ * lunr.Vector
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * A vector is used to construct the vector space of documents and queries. These
+ * vectors support operations to determine the similarity between two documents or
+ * a document and a query.
+ *
+ * Normally no parameters are required for initializing a vector, but in the case of
+ * loading a previously dumped vector the raw elements can be provided to the constructor.
+ *
+ * For performance reasons vectors are implemented with a flat array, where an elements
+ * index is immediately followed by its value. E.g. [index, value, index, value]. This
+ * allows the underlying array to be as sparse as possible and still offer decent
+ * performance when being used for vector calculations.
+ *
+ * @constructor
+ * @param {Number[]} [elements] - The flat list of element index and element value pairs.
+ */
+lunr.Vector = function (elements) {
+ this._magnitude = 0
+ this.elements = elements || []
+}
+
+
+/**
+ * Calculates the position within the vector to insert a given index.
+ *
+ * This is used internally by insert and upsert. If there are duplicate indexes then
+ * the position is returned as if the value for that index were to be updated, but it
+ * is the callers responsibility to check whether there is a duplicate at that index
+ *
+ * @param {Number} insertIdx - The index at which the element should be inserted.
+ * @returns {Number}
+ */
+lunr.Vector.prototype.positionForIndex = function (index) {
+ // For an empty vector the tuple can be inserted at the beginning
+ if (this.elements.length == 0) {
+ return 0
+ }
+
+ var start = 0,
+ end = this.elements.length / 2,
+ sliceLength = end - start,
+ pivotPoint = Math.floor(sliceLength / 2),
+ pivotIndex = this.elements[pivotPoint * 2]
+
+ while (sliceLength > 1) {
+ if (pivotIndex < index) {
+ start = pivotPoint
+ }
+
+ if (pivotIndex > index) {
+ end = pivotPoint
+ }
+
+ if (pivotIndex == index) {
+ break
+ }
+
+ sliceLength = end - start
+ pivotPoint = start + Math.floor(sliceLength / 2)
+ pivotIndex = this.elements[pivotPoint * 2]
+ }
+
+ if (pivotIndex == index) {
+ return pivotPoint * 2
+ }
+
+ if (pivotIndex > index) {
+ return pivotPoint * 2
+ }
+
+ if (pivotIndex < index) {
+ return (pivotPoint + 1) * 2
+ }
+}
+
+/**
+ * Inserts an element at an index within the vector.
+ *
+ * Does not allow duplicates, will throw an error if there is already an entry
+ * for this index.
+ *
+ * @param {Number} insertIdx - The index at which the element should be inserted.
+ * @param {Number} val - The value to be inserted into the vector.
+ */
+lunr.Vector.prototype.insert = function (insertIdx, val) {
+ this.upsert(insertIdx, val, function () {
+ throw "duplicate index"
+ })
+}
+
+/**
+ * Inserts or updates an existing index within the vector.
+ *
+ * @param {Number} insertIdx - The index at which the element should be inserted.
+ * @param {Number} val - The value to be inserted into the vector.
+ * @param {function} fn - A function that is called for updates, the existing value and the
+ * requested value are passed as arguments
+ */
+lunr.Vector.prototype.upsert = function (insertIdx, val, fn) {
+ this._magnitude = 0
+ var position = this.positionForIndex(insertIdx)
+
+ if (this.elements[position] == insertIdx) {
+ this.elements[position + 1] = fn(this.elements[position + 1], val)
+ } else {
+ this.elements.splice(position, 0, insertIdx, val)
+ }
+}
+
+/**
+ * Calculates the magnitude of this vector.
+ *
+ * @returns {Number}
+ */
+lunr.Vector.prototype.magnitude = function () {
+ if (this._magnitude) return this._magnitude
+
+ var sumOfSquares = 0,
+ elementsLength = this.elements.length
+
+ for (var i = 1; i < elementsLength; i += 2) {
+ var val = this.elements[i]
+ sumOfSquares += val * val
+ }
+
+ return this._magnitude = Math.sqrt(sumOfSquares)
+}
+
+/**
+ * Calculates the dot product of this vector and another vector.
+ *
+ * @param {lunr.Vector} otherVector - The vector to compute the dot product with.
+ * @returns {Number}
+ */
+lunr.Vector.prototype.dot = function (otherVector) {
+ var dotProduct = 0,
+ a = this.elements, b = otherVector.elements,
+ aLen = a.length, bLen = b.length,
+ aVal = 0, bVal = 0,
+ i = 0, j = 0
+
+ while (i < aLen && j < bLen) {
+ aVal = a[i], bVal = b[j]
+ if (aVal < bVal) {
+ i += 2
+ } else if (aVal > bVal) {
+ j += 2
+ } else if (aVal == bVal) {
+ dotProduct += a[i + 1] * b[j + 1]
+ i += 2
+ j += 2
+ }
+ }
+
+ return dotProduct
+}
+
+/**
+ * Calculates the similarity between this vector and another vector.
+ *
+ * @param {lunr.Vector} otherVector - The other vector to calculate the
+ * similarity with.
+ * @returns {Number}
+ */
+lunr.Vector.prototype.similarity = function (otherVector) {
+ return this.dot(otherVector) / this.magnitude() || 0
+}
+
+/**
+ * Converts the vector to an array of the elements within the vector.
+ *
+ * @returns {Number[]}
+ */
+lunr.Vector.prototype.toArray = function () {
+ var output = new Array (this.elements.length / 2)
+
+ for (var i = 1, j = 0; i < this.elements.length; i += 2, j++) {
+ output[j] = this.elements[i]
+ }
+
+ return output
+}
+
+/**
+ * A JSON serializable representation of the vector.
+ *
+ * @returns {Number[]}
+ */
+lunr.Vector.prototype.toJSON = function () {
+ return this.elements
+}
+/* eslint-disable */
+/*!
+ * lunr.stemmer
+ * Copyright (C) 2020 Oliver Nightingale
+ * Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt
+ */
+
+/**
+ * lunr.stemmer is an english language stemmer, this is a JavaScript
+ * implementation of the PorterStemmer taken from http://tartarus.org/~martin
+ *
+ * @static
+ * @implements {lunr.PipelineFunction}
+ * @param {lunr.Token} token - The string to stem
+ * @returns {lunr.Token}
+ * @see {@link lunr.Pipeline}
+ * @function
+ */
+lunr.stemmer = (function(){
+ var step2list = {
+ "ational" : "ate",
+ "tional" : "tion",
+ "enci" : "ence",
+ "anci" : "ance",
+ "izer" : "ize",
+ "bli" : "ble",
+ "alli" : "al",
+ "entli" : "ent",
+ "eli" : "e",
+ "ousli" : "ous",
+ "ization" : "ize",
+ "ation" : "ate",
+ "ator" : "ate",
+ "alism" : "al",
+ "iveness" : "ive",
+ "fulness" : "ful",
+ "ousness" : "ous",
+ "aliti" : "al",
+ "iviti" : "ive",
+ "biliti" : "ble",
+ "logi" : "log"
+ },
+
+ step3list = {
+ "icate" : "ic",
+ "ative" : "",
+ "alize" : "al",
+ "iciti" : "ic",
+ "ical" : "ic",
+ "ful" : "",
+ "ness" : ""
+ },
+
+ c = "[^aeiou]", // consonant
+ v = "[aeiouy]", // vowel
+ C = c + "[^aeiouy]*", // consonant sequence
+ V = v + "[aeiou]*", // vowel sequence
+
+ mgr0 = "^(" + C + ")?" + V + C, // [C]VC... is m>0
+ meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$", // [C]VC[V] is m=1
+ mgr1 = "^(" + C + ")?" + V + C + V + C, // [C]VCVC... is m>1
+ s_v = "^(" + C + ")?" + v; // vowel in stem
+
+ var re_mgr0 = new RegExp(mgr0);
+ var re_mgr1 = new RegExp(mgr1);
+ var re_meq1 = new RegExp(meq1);
+ var re_s_v = new RegExp(s_v);
+
+ var re_1a = /^(.+?)(ss|i)es$/;
+ var re2_1a = /^(.+?)([^s])s$/;
+ var re_1b = /^(.+?)eed$/;
+ var re2_1b = /^(.+?)(ed|ing)$/;
+ var re_1b_2 = /.$/;
+ var re2_1b_2 = /(at|bl|iz)$/;
+ var re3_1b_2 = new RegExp("([^aeiouylsz])\\1$");
+ var re4_1b_2 = new RegExp("^" + C + v + "[^aeiouwxy]$");
+
+ var re_1c = /^(.+?[^aeiou])y$/;
+ var re_2 = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;
+
+ var re_3 = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/;
+
+ var re_4 = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/;
+ var re2_4 = /^(.+?)(s|t)(ion)$/;
+
+ var re_5 = /^(.+?)e$/;
+ var re_5_1 = /ll$/;
+ var re3_5 = new RegExp("^" + C + v + "[^aeiouwxy]$");
+
+ var porterStemmer = function porterStemmer(w) {
+ var stem,
+ suffix,
+ firstch,
+ re,
+ re2,
+ re3,
+ re4;
+
+ if (w.length < 3) { return w; }
+
+ firstch = w.substr(0,1);
+ if (firstch == "y") {
+ w = firstch.toUpperCase() + w.substr(1);
+ }
+
+ // Step 1a
+ re = re_1a
+ re2 = re2_1a;
+
+ if (re.test(w)) { w = w.replace(re,"$1$2"); }
+ else if (re2.test(w)) { w = w.replace(re2,"$1$2"); }
+
+ // Step 1b
+ re = re_1b;
+ re2 = re2_1b;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ re = re_mgr0;
+ if (re.test(fp[1])) {
+ re = re_1b_2;
+ w = w.replace(re,"");
+ }
+ } else if (re2.test(w)) {
+ var fp = re2.exec(w);
+ stem = fp[1];
+ re2 = re_s_v;
+ if (re2.test(stem)) {
+ w = stem;
+ re2 = re2_1b_2;
+ re3 = re3_1b_2;
+ re4 = re4_1b_2;
+ if (re2.test(w)) { w = w + "e"; }
+ else if (re3.test(w)) { re = re_1b_2; w = w.replace(re,""); }
+ else if (re4.test(w)) { w = w + "e"; }
+ }
+ }
+
+ // Step 1c - replace suffix y or Y by i if preceded by a non-vowel which is not the first letter of the word (so cry -> cri, by -> by, say -> say)
+ re = re_1c;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ w = stem + "i";
+ }
+
+ // Step 2
+ re = re_2;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ suffix = fp[2];
+ re = re_mgr0;
+ if (re.test(stem)) {
+ w = stem + step2list[suffix];
+ }
+ }
+
+ // Step 3
+ re = re_3;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ suffix = fp[2];
+ re = re_mgr0;
+ if (re.test(stem)) {
+ w = stem + step3list[suffix];
+ }
+ }
+
+ // Step 4
+ re = re_4;
+ re2 = re2_4;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ re = re_mgr1;
+ if (re.test(stem)) {
+ w = stem;
+ }
+ } else if (re2.test(w)) {
+ var fp = re2.exec(w);
+ stem = fp[1] + fp[2];
+ re2 = re_mgr1;
+ if (re2.test(stem)) {
+ w = stem;
+ }
+ }
+
+ // Step 5
+ re = re_5;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ re = re_mgr1;
+ re2 = re_meq1;
+ re3 = re3_5;
+ if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) {
+ w = stem;
+ }
+ }
+
+ re = re_5_1;
+ re2 = re_mgr1;
+ if (re.test(w) && re2.test(w)) {
+ re = re_1b_2;
+ w = w.replace(re,"");
+ }
+
+ // and turn initial Y back to y
+
+ if (firstch == "y") {
+ w = firstch.toLowerCase() + w.substr(1);
+ }
+
+ return w;
+ };
+
+ return function (token) {
+ return token.update(porterStemmer);
+ }
+})();
+
+lunr.Pipeline.registerFunction(lunr.stemmer, 'stemmer')
+/*!
+ * lunr.stopWordFilter
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * lunr.generateStopWordFilter builds a stopWordFilter function from the provided
+ * list of stop words.
+ *
+ * The built in lunr.stopWordFilter is built using this generator and can be used
+ * to generate custom stopWordFilters for applications or non English languages.
+ *
+ * @function
+ * @param {Array} token The token to pass through the filter
+ * @returns {lunr.PipelineFunction}
+ * @see lunr.Pipeline
+ * @see lunr.stopWordFilter
+ */
+lunr.generateStopWordFilter = function (stopWords) {
+ var words = stopWords.reduce(function (memo, stopWord) {
+ memo[stopWord] = stopWord
+ return memo
+ }, {})
+
+ return function (token) {
+ if (token && words[token.toString()] !== token.toString()) return token
+ }
+}
+
+/**
+ * lunr.stopWordFilter is an English language stop word list filter, any words
+ * contained in the list will not be passed through the filter.
+ *
+ * This is intended to be used in the Pipeline. If the token does not pass the
+ * filter then undefined will be returned.
+ *
+ * @function
+ * @implements {lunr.PipelineFunction}
+ * @params {lunr.Token} token - A token to check for being a stop word.
+ * @returns {lunr.Token}
+ * @see {@link lunr.Pipeline}
+ */
+lunr.stopWordFilter = lunr.generateStopWordFilter([
+ 'a',
+ 'able',
+ 'about',
+ 'across',
+ 'after',
+ 'all',
+ 'almost',
+ 'also',
+ 'am',
+ 'among',
+ 'an',
+ 'and',
+ 'any',
+ 'are',
+ 'as',
+ 'at',
+ 'be',
+ 'because',
+ 'been',
+ 'but',
+ 'by',
+ 'can',
+ 'cannot',
+ 'could',
+ 'dear',
+ 'did',
+ 'do',
+ 'does',
+ 'either',
+ 'else',
+ 'ever',
+ 'every',
+ 'for',
+ 'from',
+ 'get',
+ 'got',
+ 'had',
+ 'has',
+ 'have',
+ 'he',
+ 'her',
+ 'hers',
+ 'him',
+ 'his',
+ 'how',
+ 'however',
+ 'i',
+ 'if',
+ 'in',
+ 'into',
+ 'is',
+ 'it',
+ 'its',
+ 'just',
+ 'least',
+ 'let',
+ 'like',
+ 'likely',
+ 'may',
+ 'me',
+ 'might',
+ 'most',
+ 'must',
+ 'my',
+ 'neither',
+ 'no',
+ 'nor',
+ 'not',
+ 'of',
+ 'off',
+ 'often',
+ 'on',
+ 'only',
+ 'or',
+ 'other',
+ 'our',
+ 'own',
+ 'rather',
+ 'said',
+ 'say',
+ 'says',
+ 'she',
+ 'should',
+ 'since',
+ 'so',
+ 'some',
+ 'than',
+ 'that',
+ 'the',
+ 'their',
+ 'them',
+ 'then',
+ 'there',
+ 'these',
+ 'they',
+ 'this',
+ 'tis',
+ 'to',
+ 'too',
+ 'twas',
+ 'us',
+ 'wants',
+ 'was',
+ 'we',
+ 'were',
+ 'what',
+ 'when',
+ 'where',
+ 'which',
+ 'while',
+ 'who',
+ 'whom',
+ 'why',
+ 'will',
+ 'with',
+ 'would',
+ 'yet',
+ 'you',
+ 'your'
+])
+
+lunr.Pipeline.registerFunction(lunr.stopWordFilter, 'stopWordFilter')
+/*!
+ * lunr.trimmer
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * lunr.trimmer is a pipeline function for trimming non word
+ * characters from the beginning and end of tokens before they
+ * enter the index.
+ *
+ * This implementation may not work correctly for non latin
+ * characters and should either be removed or adapted for use
+ * with languages with non-latin characters.
+ *
+ * @static
+ * @implements {lunr.PipelineFunction}
+ * @param {lunr.Token} token The token to pass through the filter
+ * @returns {lunr.Token}
+ * @see lunr.Pipeline
+ */
+lunr.trimmer = function (token) {
+ return token.update(function (s) {
+ return s.replace(/^\W+/, '').replace(/\W+$/, '')
+ })
+}
+
+lunr.Pipeline.registerFunction(lunr.trimmer, 'trimmer')
+/*!
+ * lunr.TokenSet
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * A token set is used to store the unique list of all tokens
+ * within an index. Token sets are also used to represent an
+ * incoming query to the index, this query token set and index
+ * token set are then intersected to find which tokens to look
+ * up in the inverted index.
+ *
+ * A token set can hold multiple tokens, as in the case of the
+ * index token set, or it can hold a single token as in the
+ * case of a simple query token set.
+ *
+ * Additionally token sets are used to perform wildcard matching.
+ * Leading, contained and trailing wildcards are supported, and
+ * from this edit distance matching can also be provided.
+ *
+ * Token sets are implemented as a minimal finite state automata,
+ * where both common prefixes and suffixes are shared between tokens.
+ * This helps to reduce the space used for storing the token set.
+ *
+ * @constructor
+ */
+lunr.TokenSet = function () {
+ this.final = false
+ this.edges = {}
+ this.id = lunr.TokenSet._nextId
+ lunr.TokenSet._nextId += 1
+}
+
+/**
+ * Keeps track of the next, auto increment, identifier to assign
+ * to a new tokenSet.
+ *
+ * TokenSets require a unique identifier to be correctly minimised.
+ *
+ * @private
+ */
+lunr.TokenSet._nextId = 1
+
+/**
+ * Creates a TokenSet instance from the given sorted array of words.
+ *
+ * @param {String[]} arr - A sorted array of strings to create the set from.
+ * @returns {lunr.TokenSet}
+ * @throws Will throw an error if the input array is not sorted.
+ */
+lunr.TokenSet.fromArray = function (arr) {
+ var builder = new lunr.TokenSet.Builder
+
+ for (var i = 0, len = arr.length; i < len; i++) {
+ builder.insert(arr[i])
+ }
+
+ builder.finish()
+ return builder.root
+}
+
+/**
+ * Creates a token set from a query clause.
+ *
+ * @private
+ * @param {Object} clause - A single clause from lunr.Query.
+ * @param {string} clause.term - The query clause term.
+ * @param {number} [clause.editDistance] - The optional edit distance for the term.
+ * @returns {lunr.TokenSet}
+ */
+lunr.TokenSet.fromClause = function (clause) {
+ if ('editDistance' in clause) {
+ return lunr.TokenSet.fromFuzzyString(clause.term, clause.editDistance)
+ } else {
+ return lunr.TokenSet.fromString(clause.term)
+ }
+}
+
+/**
+ * Creates a token set representing a single string with a specified
+ * edit distance.
+ *
+ * Insertions, deletions, substitutions and transpositions are each
+ * treated as an edit distance of 1.
+ *
+ * Increasing the allowed edit distance will have a dramatic impact
+ * on the performance of both creating and intersecting these TokenSets.
+ * It is advised to keep the edit distance less than 3.
+ *
+ * @param {string} str - The string to create the token set from.
+ * @param {number} editDistance - The allowed edit distance to match.
+ * @returns {lunr.Vector}
+ */
+lunr.TokenSet.fromFuzzyString = function (str, editDistance) {
+ var root = new lunr.TokenSet
+
+ var stack = [{
+ node: root,
+ editsRemaining: editDistance,
+ str: str
+ }]
+
+ while (stack.length) {
+ var frame = stack.pop()
+
+ // no edit
+ if (frame.str.length > 0) {
+ var char = frame.str.charAt(0),
+ noEditNode
+
+ if (char in frame.node.edges) {
+ noEditNode = frame.node.edges[char]
+ } else {
+ noEditNode = new lunr.TokenSet
+ frame.node.edges[char] = noEditNode
+ }
+
+ if (frame.str.length == 1) {
+ noEditNode.final = true
+ }
+
+ stack.push({
+ node: noEditNode,
+ editsRemaining: frame.editsRemaining,
+ str: frame.str.slice(1)
+ })
+ }
+
+ if (frame.editsRemaining == 0) {
+ continue
+ }
+
+ // insertion
+ if ("*" in frame.node.edges) {
+ var insertionNode = frame.node.edges["*"]
+ } else {
+ var insertionNode = new lunr.TokenSet
+ frame.node.edges["*"] = insertionNode
+ }
+
+ if (frame.str.length == 0) {
+ insertionNode.final = true
+ }
+
+ stack.push({
+ node: insertionNode,
+ editsRemaining: frame.editsRemaining - 1,
+ str: frame.str
+ })
+
+ // deletion
+ // can only do a deletion if we have enough edits remaining
+ // and if there are characters left to delete in the string
+ if (frame.str.length > 1) {
+ stack.push({
+ node: frame.node,
+ editsRemaining: frame.editsRemaining - 1,
+ str: frame.str.slice(1)
+ })
+ }
+
+ // deletion
+ // just removing the last character from the str
+ if (frame.str.length == 1) {
+ frame.node.final = true
+ }
+
+ // substitution
+ // can only do a substitution if we have enough edits remaining
+ // and if there are characters left to substitute
+ if (frame.str.length >= 1) {
+ if ("*" in frame.node.edges) {
+ var substitutionNode = frame.node.edges["*"]
+ } else {
+ var substitutionNode = new lunr.TokenSet
+ frame.node.edges["*"] = substitutionNode
+ }
+
+ if (frame.str.length == 1) {
+ substitutionNode.final = true
+ }
+
+ stack.push({
+ node: substitutionNode,
+ editsRemaining: frame.editsRemaining - 1,
+ str: frame.str.slice(1)
+ })
+ }
+
+ // transposition
+ // can only do a transposition if there are edits remaining
+ // and there are enough characters to transpose
+ if (frame.str.length > 1) {
+ var charA = frame.str.charAt(0),
+ charB = frame.str.charAt(1),
+ transposeNode
+
+ if (charB in frame.node.edges) {
+ transposeNode = frame.node.edges[charB]
+ } else {
+ transposeNode = new lunr.TokenSet
+ frame.node.edges[charB] = transposeNode
+ }
+
+ if (frame.str.length == 1) {
+ transposeNode.final = true
+ }
+
+ stack.push({
+ node: transposeNode,
+ editsRemaining: frame.editsRemaining - 1,
+ str: charA + frame.str.slice(2)
+ })
+ }
+ }
+
+ return root
+}
+
+/**
+ * Creates a TokenSet from a string.
+ *
+ * The string may contain one or more wildcard characters (*)
+ * that will allow wildcard matching when intersecting with
+ * another TokenSet.
+ *
+ * @param {string} str - The string to create a TokenSet from.
+ * @returns {lunr.TokenSet}
+ */
+lunr.TokenSet.fromString = function (str) {
+ var node = new lunr.TokenSet,
+ root = node
+
+ /*
+ * Iterates through all characters within the passed string
+ * appending a node for each character.
+ *
+ * When a wildcard character is found then a self
+ * referencing edge is introduced to continually match
+ * any number of any characters.
+ */
+ for (var i = 0, len = str.length; i < len; i++) {
+ var char = str[i],
+ final = (i == len - 1)
+
+ if (char == "*") {
+ node.edges[char] = node
+ node.final = final
+
+ } else {
+ var next = new lunr.TokenSet
+ next.final = final
+
+ node.edges[char] = next
+ node = next
+ }
+ }
+
+ return root
+}
+
+/**
+ * Converts this TokenSet into an array of strings
+ * contained within the TokenSet.
+ *
+ * This is not intended to be used on a TokenSet that
+ * contains wildcards, in these cases the results are
+ * undefined and are likely to cause an infinite loop.
+ *
+ * @returns {string[]}
+ */
+lunr.TokenSet.prototype.toArray = function () {
+ var words = []
+
+ var stack = [{
+ prefix: "",
+ node: this
+ }]
+
+ while (stack.length) {
+ var frame = stack.pop(),
+ edges = Object.keys(frame.node.edges),
+ len = edges.length
+
+ if (frame.node.final) {
+ /* In Safari, at this point the prefix is sometimes corrupted, see:
+ * https://github.com/olivernn/lunr.js/issues/279 Calling any
+ * String.prototype method forces Safari to "cast" this string to what
+ * it's supposed to be, fixing the bug. */
+ frame.prefix.charAt(0)
+ words.push(frame.prefix)
+ }
+
+ for (var i = 0; i < len; i++) {
+ var edge = edges[i]
+
+ stack.push({
+ prefix: frame.prefix.concat(edge),
+ node: frame.node.edges[edge]
+ })
+ }
+ }
+
+ return words
+}
+
+/**
+ * Generates a string representation of a TokenSet.
+ *
+ * This is intended to allow TokenSets to be used as keys
+ * in objects, largely to aid the construction and minimisation
+ * of a TokenSet. As such it is not designed to be a human
+ * friendly representation of the TokenSet.
+ *
+ * @returns {string}
+ */
+lunr.TokenSet.prototype.toString = function () {
+ // NOTE: Using Object.keys here as this.edges is very likely
+ // to enter 'hash-mode' with many keys being added
+ //
+ // avoiding a for-in loop here as it leads to the function
+ // being de-optimised (at least in V8). From some simple
+ // benchmarks the performance is comparable, but allowing
+ // V8 to optimize may mean easy performance wins in the future.
+
+ if (this._str) {
+ return this._str
+ }
+
+ var str = this.final ? '1' : '0',
+ labels = Object.keys(this.edges).sort(),
+ len = labels.length
+
+ for (var i = 0; i < len; i++) {
+ var label = labels[i],
+ node = this.edges[label]
+
+ str = str + label + node.id
+ }
+
+ return str
+}
+
+/**
+ * Returns a new TokenSet that is the intersection of
+ * this TokenSet and the passed TokenSet.
+ *
+ * This intersection will take into account any wildcards
+ * contained within the TokenSet.
+ *
+ * @param {lunr.TokenSet} b - An other TokenSet to intersect with.
+ * @returns {lunr.TokenSet}
+ */
+lunr.TokenSet.prototype.intersect = function (b) {
+ var output = new lunr.TokenSet,
+ frame = undefined
+
+ var stack = [{
+ qNode: b,
+ output: output,
+ node: this
+ }]
+
+ while (stack.length) {
+ frame = stack.pop()
+
+ // NOTE: As with the #toString method, we are using
+ // Object.keys and a for loop instead of a for-in loop
+ // as both of these objects enter 'hash' mode, causing
+ // the function to be de-optimised in V8
+ var qEdges = Object.keys(frame.qNode.edges),
+ qLen = qEdges.length,
+ nEdges = Object.keys(frame.node.edges),
+ nLen = nEdges.length
+
+ for (var q = 0; q < qLen; q++) {
+ var qEdge = qEdges[q]
+
+ for (var n = 0; n < nLen; n++) {
+ var nEdge = nEdges[n]
+
+ if (nEdge == qEdge || qEdge == '*') {
+ var node = frame.node.edges[nEdge],
+ qNode = frame.qNode.edges[qEdge],
+ final = node.final && qNode.final,
+ next = undefined
+
+ if (nEdge in frame.output.edges) {
+ // an edge already exists for this character
+ // no need to create a new node, just set the finality
+ // bit unless this node is already final
+ next = frame.output.edges[nEdge]
+ next.final = next.final || final
+
+ } else {
+ // no edge exists yet, must create one
+ // set the finality bit and insert it
+ // into the output
+ next = new lunr.TokenSet
+ next.final = final
+ frame.output.edges[nEdge] = next
+ }
+
+ stack.push({
+ qNode: qNode,
+ output: next,
+ node: node
+ })
+ }
+ }
+ }
+ }
+
+ return output
+}
+lunr.TokenSet.Builder = function () {
+ this.previousWord = ""
+ this.root = new lunr.TokenSet
+ this.uncheckedNodes = []
+ this.minimizedNodes = {}
+}
+
+lunr.TokenSet.Builder.prototype.insert = function (word) {
+ var node,
+ commonPrefix = 0
+
+ if (word < this.previousWord) {
+ throw new Error ("Out of order word insertion")
+ }
+
+ for (var i = 0; i < word.length && i < this.previousWord.length; i++) {
+ if (word[i] != this.previousWord[i]) break
+ commonPrefix++
+ }
+
+ this.minimize(commonPrefix)
+
+ if (this.uncheckedNodes.length == 0) {
+ node = this.root
+ } else {
+ node = this.uncheckedNodes[this.uncheckedNodes.length - 1].child
+ }
+
+ for (var i = commonPrefix; i < word.length; i++) {
+ var nextNode = new lunr.TokenSet,
+ char = word[i]
+
+ node.edges[char] = nextNode
+
+ this.uncheckedNodes.push({
+ parent: node,
+ char: char,
+ child: nextNode
+ })
+
+ node = nextNode
+ }
+
+ node.final = true
+ this.previousWord = word
+}
+
+lunr.TokenSet.Builder.prototype.finish = function () {
+ this.minimize(0)
+}
+
+lunr.TokenSet.Builder.prototype.minimize = function (downTo) {
+ for (var i = this.uncheckedNodes.length - 1; i >= downTo; i--) {
+ var node = this.uncheckedNodes[i],
+ childKey = node.child.toString()
+
+ if (childKey in this.minimizedNodes) {
+ node.parent.edges[node.char] = this.minimizedNodes[childKey]
+ } else {
+ // Cache the key for this node since
+ // we know it can't change anymore
+ node.child._str = childKey
+
+ this.minimizedNodes[childKey] = node.child
+ }
+
+ this.uncheckedNodes.pop()
+ }
+}
+/*!
+ * lunr.Index
+ * Copyright (C) 2020 Oliver Nightingale
+ */
+
+/**
+ * An index contains the built index of all documents and provides a query interface
+ * to the index.
+ *
+ * Usually instances of lunr.Index will not be created using this constructor, instead
+ * lunr.Builder should be used to construct new indexes, or lunr.Index.load should be
+ * used to load previously built and serialized indexes.
+ *
+ * @constructor
+ * @param {Object} attrs - The attributes of the built search index.
+ * @param {Object} attrs.invertedIndex - An index of term/field to document reference.
+ * @param {Object' + noResultsText + '
'); + } +} + +function doSearch () { + var query = document.getElementById('mkdocs-search-query').value; + if (query.length > min_search_length) { + if (!window.Worker) { + displayResults(search(query)); + } else { + searchWorker.postMessage({query: query}); + } + } else { + // Clear results for short queries + displayResults([]); + } +} + +function initSearch () { + var search_input = document.getElementById('mkdocs-search-query'); + if (search_input) { + search_input.addEventListener("keyup", doSearch); + } + var term = getSearchTermFromLocation(); + if (term) { + search_input.value = term; + doSearch(); + } +} + +function onWorkerMessage (e) { + if (e.data.allowSearch) { + initSearch(); + } else if (e.data.results) { + var results = e.data.results; + displayResults(results); + } else if (e.data.config) { + min_search_length = e.data.config.min_search_length-1; + } +} + +if (!window.Worker) { + console.log('Web Worker API not supported'); + // load index in main thread + $.getScript(joinUrl(base_url, "search/worker.js")).done(function () { + console.log('Loaded worker'); + init(); + window.postMessage = function (msg) { + onWorkerMessage({data: msg}); + }; + }).fail(function (jqxhr, settings, exception) { + console.error('Could not load worker.js'); + }); +} else { + // Wrap search in a web worker + var searchWorker = new Worker(joinUrl(base_url, "search/worker.js")); + searchWorker.postMessage({init: true}); + searchWorker.onmessage = onWorkerMessage; +} diff --git a/search/search_index.json b/search/search_index.json new file mode 100644 index 00000000..0c84194d --- /dev/null +++ b/search/search_index.json @@ -0,0 +1 @@ +{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Welcome to the course: Introduction to Kebnekaise \u00b6 This material Here you will find the content of the workshop \u201cIntroduction to Kebnekaise\u201d. You can download the markdown files for the presentation as well as the exercises from https://github.com/hpc2n/intro-course Click the gren \u201cCode\u201d button Either copy the url for the repo under HTTPS and do git clone https://github.com/hpc2n/intro-course.git in a terminal window OR pick \u201cDownload zip\u201d to get a zip file with the content. Some useful links: Documentation about Linux at HPC2N: https://docs.hpc2n.umu.se/tutorials/linuxguide/ Get started guide: https://docs.hpc2n.umu.se/tutorials/quickstart/ Documentation pages at HPC2N: https://docs.hpc2n.umu.se/ Prerequisites Basic knowledge about Linux (if you need a refresher, you could take the course \u201cIntroduction to Linux\u201d which runs immediately before this course. Info and registration here: https://www.hpc2n.umu.se/events/courses/2024/fall/intro-linux . An account at SUPR and at HPC2N. You should have already been contacted about getting these if you did not have them already. Content This course aims to give a brief, but comprehensive introduction to Kebnekaise. You will learn about HPC2N, HPC, and Kebnekaise hardware How to use our systems: Logging in The File System The Module System Compiler Tool Chains The Batch System Simple examples (batch system) Application examples (batch system) This course will consist of lectures and type-alongs, as well as a few exercises where you get to try out what you have just learned. Instructors \u00b6 Birgitte Bryds\u00f6, HPC2N Pedro Ojeda May, HPC2N Preliminary schedule \u00b6 Time Topic Activity 11:15 Welcome+Syllabus 11:25 Introduction to Kebnekaise and HPC2N Lecture 11:45 Logging in Lecture+code along 11:55 The File System Lecture+code along 12:15 LUNCH BREAK 13:15 The Module System Lecture+code along 13:35 Compiling, compiler tool chains Lecture+code along+exercise 13:50 The Batch System Lecture+code along 14:10 Simple Examples Lecture+exercises 14:45 COFFEE BREAK 15:00 Application Examples Lecture+code along+exercises 16:40 Questions+Summary 17:00 END OF COURSE","title":"Home"},{"location":"#welcome__to__the__course__introduction__to__kebnekaise","text":"This material Here you will find the content of the workshop \u201cIntroduction to Kebnekaise\u201d. You can download the markdown files for the presentation as well as the exercises from https://github.com/hpc2n/intro-course Click the gren \u201cCode\u201d button Either copy the url for the repo under HTTPS and do git clone https://github.com/hpc2n/intro-course.git in a terminal window OR pick \u201cDownload zip\u201d to get a zip file with the content. Some useful links: Documentation about Linux at HPC2N: https://docs.hpc2n.umu.se/tutorials/linuxguide/ Get started guide: https://docs.hpc2n.umu.se/tutorials/quickstart/ Documentation pages at HPC2N: https://docs.hpc2n.umu.se/ Prerequisites Basic knowledge about Linux (if you need a refresher, you could take the course \u201cIntroduction to Linux\u201d which runs immediately before this course. Info and registration here: https://www.hpc2n.umu.se/events/courses/2024/fall/intro-linux . An account at SUPR and at HPC2N. You should have already been contacted about getting these if you did not have them already. Content This course aims to give a brief, but comprehensive introduction to Kebnekaise. You will learn about HPC2N, HPC, and Kebnekaise hardware How to use our systems: Logging in The File System The Module System Compiler Tool Chains The Batch System Simple examples (batch system) Application examples (batch system) This course will consist of lectures and type-alongs, as well as a few exercises where you get to try out what you have just learned.","title":"Welcome to the course: Introduction to Kebnekaise"},{"location":"#instructors","text":"Birgitte Bryds\u00f6, HPC2N Pedro Ojeda May, HPC2N","title":"Instructors"},{"location":"#preliminary__schedule","text":"Time Topic Activity 11:15 Welcome+Syllabus 11:25 Introduction to Kebnekaise and HPC2N Lecture 11:45 Logging in Lecture+code along 11:55 The File System Lecture+code along 12:15 LUNCH BREAK 13:15 The Module System Lecture+code along 13:35 Compiling, compiler tool chains Lecture+code along+exercise 13:50 The Batch System Lecture+code along 14:10 Simple Examples Lecture+exercises 14:45 COFFEE BREAK 15:00 Application Examples Lecture+code along+exercises 16:40 Questions+Summary 17:00 END OF COURSE","title":"Preliminary schedule"},{"location":"intro/","text":"Introduction to HPC2N, Kebnekaise and HPC \u00b6 Welcome page and syllabus: https://hpc2n.github.io/intro-linux/index.html Also link at the House symbol at the top of the page. HPC2N \u00b6 Note High Performance Computing Center North (HPC2N) is a competence center for Scientific and Parallel Computing part of National Academic Infrastructure for Super\u00adcomputing in Sweden (NAISS) HPC2N provides state-of-the-art resources and expertise: Scalable and parallel HPC Large-scale storage facilities (Project storage (Lustre), SweStore, Tape) Grid and cloud computing (WLCG NT1, Swedish Science Cloud) National Data Science Node in \u201dEpidemiology and Biology of Infections\u201d (DDLS) Software for e-Science applications All levels of user support Primary, advanced, dedicated Application Experts (AEs) Primary objective To raise the national and local level of HPC competence and transfer HPC knowledge and technology to new users in academia and industry. HPC2N partners \u00b6 HPC2N is hosted by: Partners: HPC2N funding and collaborations \u00b6 Funded mainly by Ume\u00e5 University , with contributions from the other HPC2N partners . Involved in several projects and collaborations : HPC2N training and other services \u00b6 User support (primary, advanced, dedicated) Research group meetings @ UmU Also at the partner sites Online \u201cHPC2N fika\u201d User training and education program 0.5 \u2013 5 days; ready-to-run exercises Introduction to Linux, 16 September 2024 Introduction to HPC2N and Kebnekaise, 16 September 2024 Parallel programming and tools (OpenMP, MPI, debugging, perf. analyzers, Matlab, R, MD simulation, ML, GPU, \u2026) Basic Singularity, 16 October 2024 Introduction to running R, Python, Julia, and Matlab in HPC, 22-25 October 2024 Introduction to Git, 25-29 November 2024 Using Python in an HPC environment, 5-6 December 2024 Updated list: https://www.hpc2n.umu.se/events/courses Workshops and seminars NGSSC / SeSE & university courses HPC2N personnel \u00b6 Management: Paolo Bientinesi, director Bj\u00f6rn Torkelsson, deputy director Lena Hellman, administrator Application experts: Jerry Eriksson Pedro Ojeda May Birgitte Bryds\u00f6 \u00c5ke Sandgren Others: Mikael R\u00e4nnar (WLCG coord) Research Engineers under DDLS, HPC2N/SciLifeLab Paul Dulaud, System Developer, IT Abdullah Aziz, Data Engineer Data Steward System and support: Erik Andersson Birgitte Bryds\u00f6 Niklas Edmundsson (Tape coord) My Karlsson Roger Oscarsson \u00c5ke Sandgren Mattias Wadenstein (NeIC, Tier1) Lars Viklund HPC2N application experts \u00b6 HPC2N provides advanced and dedicated support in the form of Application Experts (AEs) : Jerry Eriksson: Profiling, Machine learning (DNN), MPI, OpenMP, OpenACC Pedro Ojeda May: Molecular dynamics, Profiling, QM/MM, NAMD, Amber, Gromacs, GAUSSIAN, R, Python \u00c5ke Sandgren: General high level programming assistance, VASP, Gromacs, Amber Birgitte Bryds\u00f6: General HPC, R, Python Contact through regular support HPC2N users by discipline \u00b6 Users from several scientific disciplines: Biosciences and medicine Chemistry Computing science Engineering Materials science Mathematics and statistics Physics including space physics ML, DL, and other AI HPC2N users by discipline, largest users \u00b6 Users from several scientific disciplines: Biosciences and medicine Chemistry Computing science Engineering Materials science Mathematics and statistics Physics including space physics Machine learning and artificial intelligence (several new projects) HPC2N users by software \u00b6 Kebnekaise \u00b6 The current supercomputer at HPC2N. It is a very heterogeneous system. Named after a massif (contains some of Sweden\u2019s highest mountain peaks) Kebnekaise was delivered by Lenovo and installed during the summer 2016 Opened up for general availability on November 7, 2016 In 2018, Kebnekaise was extended with 52 Intel Xeon Gold 6132 (Skylake) nodes, as well as 10 NVidian V100 (Volta) GPU nodes In 2023, Kebnekaise was extended with 2 dual NVIDIA A100 GPU nodes one many-core AMD Zen3 CPU node Kebnekaise will be continuosly upgraded, as old hardware gets retired. In 2024 Kebnekaise was extended with 2 Dual socket GPU-nodes: Lenovo ThinkSystem SR675 V3 2 x AMD EPYC 9454 48C 290W 2.75GHz Processor 768GB [24x 32GB TruDDR5 4800MHz RDIMM-A] 1 x 3.84TB Read Intensive NVMe PCIe 4.0 x4 HS SSD 1 x NVIDIA H100 SXM5 700W 80G HBM3 GPU Board 10 dual-socket GPU-nodes: ThinkSystem SR665 V3 2 x AMD EPYC 9254 24C 200W 2.9GHz Processor 384GB [24x 16GB TruDDR5 4800MHz RDIMM-A] 1 x 1.92TB Read Intensive NVMe PCIe 5.0 x4 HS SSD 2 x NVIDIA L40S 48GB PCIe Gen4 Passive GPU 8 dual-socket CPU only: ThinkSystem SR645 V3 2 x AMD EPYC 9754 128C 360W 2.25GHz Processor 768GB [24x 32GB TruDDR5 4800MHz RDIMM-A] 1 x 1 3.84TB Read Intensive NVMe PCIe 4.0 x4 HS SSD Current hardware in Kebnekaise \u00b6 Kebnekaise have CPU-only, GPU enabled and large memory nodes. The CPU-only nodes are: 2 x 14 core Intel broadwell 4460 MB memory / core 48 nodes Total of 41.6 TFlops/s 2 x 14 core Intel skylake 6785 MB memory / core 52 nodes Total of 87 TFlops/s 2 x 64 core AMD zen3 8020 MB / core 1 node Total of 11 TFlops/s 2 x 128 core AMD zen4 2516 MB / core 8 nodes Total of 216 TFlops/s The GPU enabled nodes are: 2 x 14 core Intel broadwell 9000 MB memory / core 2 x Nvidia A40 4 nodes Total of 83 TFlops/s 2 x 14 core Intel skylake 6785 MB memory / core 2 x Nvidia V100 10 nodes Total of 75 TFlops/s 2 x 24 core AMD zen3 10600 MB / core 2 x Nvidia A100 2 nodes 2 x 24 core AMD zen3 10600 MB / core 2 x AMD MI100 1 node 2 x 24 core AMD zen4 6630 MB / core 2 x Nvidia A6000 1 node 2 x 24 core AMD zen4 6630 MB / core 2 x Nvidia L40s 10 nodes 2 x 48 core AMD zen4 6630 MB / core 4 x Nvidia H100 SXM5 2 nodes The large memory nodes are: 4 x 18 core Intel broadwell 41666 MB memory / core 8 nodes Total of 13.6 TFlops/s for all these nodes Kebnekaise - HPC2N storage \u00b6 Basically four types of storage are available at HPC2N: Home directory /home/X/Xyz , $HOME , ~ 25 GB, user owned Project storage /proj/nobackup/abc Shared among project members Local scratch space $SNIC_TMP SSD (170GB), per job, per node, \u201cvolatile\u201d Tape Storage Backup Long term storage Also SweStore \u2014 disk based (dCache) Research Data Storage Infrastructure, for active research data and operated by NAISS, WLCG Kebnekaise - projects \u00b6 Compute projects To use Kebnekaise, you must be a member of a compute project . A compute project has a certain number of core hours allocated for it per month A regular CPU core cost 1 core hour per hour, other resources (e.g., GPUs) cost more Not a hard limit but projects that go over the allocation get lower priority A compute project contains a certain amount of storage. If more storage is required, you must be a member of a storage project . Note As Kebnekaise is a local cluster, you need to be affiliated with UmU, IRF, SLU, Miun, or LTU to use it. Projects are applied for through SUPR ( https://supr.naiss.se ). I will cover more details in a later section, where we go more into detail about HPC2N and Kebnekaise. HPC \u00b6 What is HPC? High Performance Computing (definition) \u201cHigh Performance Computing most generally refers to the practice of aggregating computing power in a way that delivers much higher performance than one could get out of a typical desktop computer or workstation in order to solve large problems in science, engineering, or business.\u201d From: https://insidehpc.com/hpc-basic-training/what-is-hpc/ High Performance Computing - opening the definition \u00b6 Aggregating computing power \u00b6 147 nodes totalling 6808 cores and ??? CUDA cores Compared to 4 cores in a modern laptop Higher performance \u00b6 More than 527,000,000,000,000 arithmetical operations per second (527 trillion (billion) Compared to 200,000,000,000 Flops in a modern laptop (200 billion (milliard) Solve large problems \u00b6 When does a problem become large enough for HPC? Are there other reasons for using HPC resources? (Memory, software, support, etc.) High Performance Computing - large problems \u00b6 A problem can be large for two main reasons: Execution time : The time required to form a solution to the problem is very long Memory / storage use : The solution of the problem requires a lot of memory and/or storage The former can be remedied by increasing the performance More cores, more nodes, GPUs, \u2026 The latter by adding more memory / storage More memory per node (including large memory nodes), more nodes, \u2026 Kebnekaise: 128GB - 192GB, 384GB, 512GB, 768GB, 3TB Large storage solutions, \u2026 High Performance Computing - what counts as HPC \u00b6 High Performance Computing - other reasons \u00b6 Specialized (expensive) hardware GPUs, including those optimized for AI Kebnekaise has V100, A100, A40, MI100, A6000, L40S, H100 High-end CPUs (AVX-512 etc) and ECC memory Software HPC2N holds licenses for several softwares Software is pre-configured and ready-to-use Support and documentation High Performance Computing - memory models \u00b6 Two memory models are relevant for HPC: Shared memory: Single memory space for all data. Everyone can access the same data Straightforward to use Distributed memory: Multiple distinct memory spaces. Everyone has direct access only to the local data Requires communication \\frametitle{High Performance Computing (memory models)} \\begin{center} \\includegraphics[width=0.6\\textwidth]{figures/memory.pdf} \\end{center} } \\frame{ \\frametitle{High Performance Computing (programming models)} \\begin{itemize} \\item The programming model changes when we aim for extra performance and/or memory: \\begin{enumerate} \\pause \\item {\\color{cyan} Single-core}: Matlab, Python, C, Fortran, \\dots \\begin{itemize} \\item Single stream of operations \\end{itemize} \\pause \\item {\\color{cyan} Multi-core}: Vectorized Matlab, pthreads, \\emph{OpenMP} \\begin{itemize} \\item \\emph{Multiple streams} of operations \\pause \\item \\emph{Work distribution}, \\emph{coordination} (synchronization, etc), \\dots \\end{itemize} \\pause \\item {\\color{cyan} Distributed memory}: \\emph{MPI}, \\dots \\begin{itemize} \\item Multiple streams of operations \\item Work distribution, coordination (synchronization, etc), \\dots \\pause \\item \\emph{Data distribution and communication} \\end{itemize} \\end{enumerate} \\pause \\item {\\color{cyan} GPUs}: \\emph{CUDA}, OpenCL, OpenACC, OpenMP, \\dots \\begin{itemize} \\pause \\item \\emph{Many lightweight} streams of operations \\pause \\item Work distribution, coordination (synchronization, etc), \\dots \\pause \\item \\emph{Data distribution across memory spaces and movement} \\end{itemize} \\end{itemize} } \\frame{ \\frametitle{High Performance Computing (software)} \\begin{itemize} \\item Complexity grows when we aim for extra performance and/or memory/storage: \\begin{enumerate} \\pause \\item {\\color{cyan} Single-core}: LAPACK, \\dots \\begin{itemize} \\item Load correct toolchain etc \\end{itemize} \\pause \\item {\\color{cyan} Multi-core}: LAPACK + parallel BLAS, \\dots \\begin{itemize} \\item Load correct toolchain etc \\pause \\item \\emph{Allocate} correct number of cores, \\emph{configure} software to use correct number of cores, \\dots \\end{itemize} \\pause \\item {\\color{cyan} Distributed memory}: ScaLAPACK, \\dots \\begin{itemize} \\item Load correct toolchain etc \\pause \\item Allocate correct number of \\emph{nodes and cores}, configure software to use correct number of \\emph{nodes and cores}, \\dots \\pause \\item Data distribution, storage, \\dots \\end{itemize} \\end{enumerate} \\pause \\item {\\color{cyan} GPUs}: MAGMA, TensorFlow, \\dots \\begin{itemize} \\item Load correct toolchain etc \\item Allocate correct number of \\emph{cores and GPUs}, configure software to use correct number of \\emph{cores and GPUs}, \\dots \\end{itemize} \\end{itemize} } \\frame{ \\frametitle{End (questions?)} \\centering \\Huge Questions? } \\end{document} Learning outcomes What is Linux? Learn about the command line interface (CLI) Navigate the file system - ls, mkdir, cd, rm, cp, mv, redirection, pipes, etc. Data handling - compressing, archiving, file transfer, patterns, etc. Hints and tricks of Linux Why Linux \u00b6 Most HPC centers supercomputers run some flavour of Linux. It is stable and reliable Customizable Runs on any hardware Strong community Many flavours are open source and free Lots of applications Lightweight What is Linux \u00b6 Linux is a family of open-source Unix-like operating systems based on the Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds. An operating system is the software that sits underneath all of the other software on a computer, managing the computer\u2019s hardware (CPU, GPU, memory, storage\u2026) and taking care of the connections between your other software and the hardware. Linux is typically packaged as a Linux distribution, which includes the kernel and supporting system software and libraries, many of which are provided by the GNU Project. Distributions (distros) There are many Linux distribuitions, including Ubuntu, Debian, Fedora, Gentoo, and many others. Many distributions are free and open source, but there are also commercial distributions, like Red Hat Enterprise and SUSE. Desktop Linux distributions include a desktop environment , like GNOME, MATE, KDE Plasma, Xfce, Unity, or many others. A window manager together with applications written using a widget toolkit are generally responsible for most of what the user sees. In addition, a windowing system of some sort (X11, Wayland) interfaces directly with the underlying operating system and libraries, providing support for graphical hardware, pointing devices, and keyboards. The window manager generally runs on top of this windowing system. While the use of Linux on desktop is only 2-3%, the vast majority of web servers (>96%), most mobile devices (Android is based on the Linux kernel), and all supercomputers on the Top500 list run Linux. More information \u00b6 There is much more information about Linux on Wikipedia . Some pages with guides and/or cheat sheets: The Linux Handbook https://www.geeksforgeeks.org/linux-tutorial/ 14 Free Training Courses to Learn Linux Online Introduction to Linux - A Hands on Guide Linux Fundamentals Top 50+ Linux Commands You MUST Know","title":"Introduction to Kebnekaise and HPC2N"},{"location":"intro/#introduction__to__hpc2n__kebnekaise__and__hpc","text":"Welcome page and syllabus: https://hpc2n.github.io/intro-linux/index.html Also link at the House symbol at the top of the page.","title":"Introduction to HPC2N, Kebnekaise and HPC"},{"location":"intro/#hpc2n","text":"Note High Performance Computing Center North (HPC2N) is a competence center for Scientific and Parallel Computing part of National Academic Infrastructure for Super\u00adcomputing in Sweden (NAISS) HPC2N provides state-of-the-art resources and expertise: Scalable and parallel HPC Large-scale storage facilities (Project storage (Lustre), SweStore, Tape) Grid and cloud computing (WLCG NT1, Swedish Science Cloud) National Data Science Node in \u201dEpidemiology and Biology of Infections\u201d (DDLS) Software for e-Science applications All levels of user support Primary, advanced, dedicated Application Experts (AEs) Primary objective To raise the national and local level of HPC competence and transfer HPC knowledge and technology to new users in academia and industry.","title":"HPC2N"},{"location":"intro/#hpc2n__partners","text":"HPC2N is hosted by: Partners:","title":"HPC2N partners"},{"location":"intro/#hpc2n__funding__and__collaborations","text":"Funded mainly by Ume\u00e5 University , with contributions from the other HPC2N partners . Involved in several projects and collaborations :","title":"HPC2N funding and collaborations"},{"location":"intro/#hpc2n__training__and__other__services","text":"User support (primary, advanced, dedicated) Research group meetings @ UmU Also at the partner sites Online \u201cHPC2N fika\u201d User training and education program 0.5 \u2013 5 days; ready-to-run exercises Introduction to Linux, 16 September 2024 Introduction to HPC2N and Kebnekaise, 16 September 2024 Parallel programming and tools (OpenMP, MPI, debugging, perf. analyzers, Matlab, R, MD simulation, ML, GPU, \u2026) Basic Singularity, 16 October 2024 Introduction to running R, Python, Julia, and Matlab in HPC, 22-25 October 2024 Introduction to Git, 25-29 November 2024 Using Python in an HPC environment, 5-6 December 2024 Updated list: https://www.hpc2n.umu.se/events/courses Workshops and seminars NGSSC / SeSE & university courses","title":"HPC2N training and other services"},{"location":"intro/#hpc2n__personnel","text":"Management: Paolo Bientinesi, director Bj\u00f6rn Torkelsson, deputy director Lena Hellman, administrator Application experts: Jerry Eriksson Pedro Ojeda May Birgitte Bryds\u00f6 \u00c5ke Sandgren Others: Mikael R\u00e4nnar (WLCG coord) Research Engineers under DDLS, HPC2N/SciLifeLab Paul Dulaud, System Developer, IT Abdullah Aziz, Data Engineer Data Steward System and support: Erik Andersson Birgitte Bryds\u00f6 Niklas Edmundsson (Tape coord) My Karlsson Roger Oscarsson \u00c5ke Sandgren Mattias Wadenstein (NeIC, Tier1) Lars Viklund","title":"HPC2N personnel"},{"location":"intro/#hpc2n__application__experts","text":"HPC2N provides advanced and dedicated support in the form of Application Experts (AEs) : Jerry Eriksson: Profiling, Machine learning (DNN), MPI, OpenMP, OpenACC Pedro Ojeda May: Molecular dynamics, Profiling, QM/MM, NAMD, Amber, Gromacs, GAUSSIAN, R, Python \u00c5ke Sandgren: General high level programming assistance, VASP, Gromacs, Amber Birgitte Bryds\u00f6: General HPC, R, Python Contact through regular support","title":"HPC2N application experts"},{"location":"intro/#hpc2n__users__by__discipline","text":"Users from several scientific disciplines: Biosciences and medicine Chemistry Computing science Engineering Materials science Mathematics and statistics Physics including space physics ML, DL, and other AI","title":"HPC2N users by discipline"},{"location":"intro/#hpc2n__users__by__discipline__largest__users","text":"Users from several scientific disciplines: Biosciences and medicine Chemistry Computing science Engineering Materials science Mathematics and statistics Physics including space physics Machine learning and artificial intelligence (several new projects)","title":"HPC2N users by discipline, largest users"},{"location":"intro/#hpc2n__users__by__software","text":"","title":"HPC2N users by software"},{"location":"intro/#kebnekaise","text":"The current supercomputer at HPC2N. It is a very heterogeneous system. Named after a massif (contains some of Sweden\u2019s highest mountain peaks) Kebnekaise was delivered by Lenovo and installed during the summer 2016 Opened up for general availability on November 7, 2016 In 2018, Kebnekaise was extended with 52 Intel Xeon Gold 6132 (Skylake) nodes, as well as 10 NVidian V100 (Volta) GPU nodes In 2023, Kebnekaise was extended with 2 dual NVIDIA A100 GPU nodes one many-core AMD Zen3 CPU node Kebnekaise will be continuosly upgraded, as old hardware gets retired. In 2024 Kebnekaise was extended with 2 Dual socket GPU-nodes: Lenovo ThinkSystem SR675 V3 2 x AMD EPYC 9454 48C 290W 2.75GHz Processor 768GB [24x 32GB TruDDR5 4800MHz RDIMM-A] 1 x 3.84TB Read Intensive NVMe PCIe 4.0 x4 HS SSD 1 x NVIDIA H100 SXM5 700W 80G HBM3 GPU Board 10 dual-socket GPU-nodes: ThinkSystem SR665 V3 2 x AMD EPYC 9254 24C 200W 2.9GHz Processor 384GB [24x 16GB TruDDR5 4800MHz RDIMM-A] 1 x 1.92TB Read Intensive NVMe PCIe 5.0 x4 HS SSD 2 x NVIDIA L40S 48GB PCIe Gen4 Passive GPU 8 dual-socket CPU only: ThinkSystem SR645 V3 2 x AMD EPYC 9754 128C 360W 2.25GHz Processor 768GB [24x 32GB TruDDR5 4800MHz RDIMM-A] 1 x 1 3.84TB Read Intensive NVMe PCIe 4.0 x4 HS SSD","title":"Kebnekaise"},{"location":"intro/#current__hardware__in__kebnekaise","text":"Kebnekaise have CPU-only, GPU enabled and large memory nodes. The CPU-only nodes are: 2 x 14 core Intel broadwell 4460 MB memory / core 48 nodes Total of 41.6 TFlops/s 2 x 14 core Intel skylake 6785 MB memory / core 52 nodes Total of 87 TFlops/s 2 x 64 core AMD zen3 8020 MB / core 1 node Total of 11 TFlops/s 2 x 128 core AMD zen4 2516 MB / core 8 nodes Total of 216 TFlops/s The GPU enabled nodes are: 2 x 14 core Intel broadwell 9000 MB memory / core 2 x Nvidia A40 4 nodes Total of 83 TFlops/s 2 x 14 core Intel skylake 6785 MB memory / core 2 x Nvidia V100 10 nodes Total of 75 TFlops/s 2 x 24 core AMD zen3 10600 MB / core 2 x Nvidia A100 2 nodes 2 x 24 core AMD zen3 10600 MB / core 2 x AMD MI100 1 node 2 x 24 core AMD zen4 6630 MB / core 2 x Nvidia A6000 1 node 2 x 24 core AMD zen4 6630 MB / core 2 x Nvidia L40s 10 nodes 2 x 48 core AMD zen4 6630 MB / core 4 x Nvidia H100 SXM5 2 nodes The large memory nodes are: 4 x 18 core Intel broadwell 41666 MB memory / core 8 nodes Total of 13.6 TFlops/s for all these nodes","title":"Current hardware in Kebnekaise"},{"location":"intro/#kebnekaise__-__hpc2n__storage","text":"Basically four types of storage are available at HPC2N: Home directory /home/X/Xyz , $HOME , ~ 25 GB, user owned Project storage /proj/nobackup/abc Shared among project members Local scratch space $SNIC_TMP SSD (170GB), per job, per node, \u201cvolatile\u201d Tape Storage Backup Long term storage Also SweStore \u2014 disk based (dCache) Research Data Storage Infrastructure, for active research data and operated by NAISS, WLCG","title":"Kebnekaise - HPC2N storage"},{"location":"intro/#kebnekaise__-__projects","text":"Compute projects To use Kebnekaise, you must be a member of a compute project . A compute project has a certain number of core hours allocated for it per month A regular CPU core cost 1 core hour per hour, other resources (e.g., GPUs) cost more Not a hard limit but projects that go over the allocation get lower priority A compute project contains a certain amount of storage. If more storage is required, you must be a member of a storage project . Note As Kebnekaise is a local cluster, you need to be affiliated with UmU, IRF, SLU, Miun, or LTU to use it. Projects are applied for through SUPR ( https://supr.naiss.se ). I will cover more details in a later section, where we go more into detail about HPC2N and Kebnekaise.","title":"Kebnekaise - projects"},{"location":"intro/#hpc","text":"What is HPC? High Performance Computing (definition) \u201cHigh Performance Computing most generally refers to the practice of aggregating computing power in a way that delivers much higher performance than one could get out of a typical desktop computer or workstation in order to solve large problems in science, engineering, or business.\u201d From: https://insidehpc.com/hpc-basic-training/what-is-hpc/","title":"HPC"},{"location":"intro/#high__performance__computing__-__opening__the__definition","text":"","title":"High Performance Computing - opening the definition"},{"location":"intro/#aggregating__computing__power","text":"147 nodes totalling 6808 cores and ??? CUDA cores Compared to 4 cores in a modern laptop","title":"Aggregating computing power"},{"location":"intro/#higher__performance","text":"More than 527,000,000,000,000 arithmetical operations per second (527 trillion (billion) Compared to 200,000,000,000 Flops in a modern laptop (200 billion (milliard)","title":"Higher performance"},{"location":"intro/#solve__large__problems","text":"When does a problem become large enough for HPC? Are there other reasons for using HPC resources? (Memory, software, support, etc.)","title":"Solve large problems"},{"location":"intro/#high__performance__computing__-__large__problems","text":"A problem can be large for two main reasons: Execution time : The time required to form a solution to the problem is very long Memory / storage use : The solution of the problem requires a lot of memory and/or storage The former can be remedied by increasing the performance More cores, more nodes, GPUs, \u2026 The latter by adding more memory / storage More memory per node (including large memory nodes), more nodes, \u2026 Kebnekaise: 128GB - 192GB, 384GB, 512GB, 768GB, 3TB Large storage solutions, \u2026","title":"High Performance Computing - large problems"},{"location":"intro/#high__performance__computing__-__what__counts__as__hpc","text":"","title":"High Performance Computing - what counts as HPC"},{"location":"intro/#high__performance__computing__-__other__reasons","text":"Specialized (expensive) hardware GPUs, including those optimized for AI Kebnekaise has V100, A100, A40, MI100, A6000, L40S, H100 High-end CPUs (AVX-512 etc) and ECC memory Software HPC2N holds licenses for several softwares Software is pre-configured and ready-to-use Support and documentation","title":"High Performance Computing - other reasons"},{"location":"intro/#high__performance__computing__-__memory__models","text":"Two memory models are relevant for HPC: Shared memory: Single memory space for all data. Everyone can access the same data Straightforward to use Distributed memory: Multiple distinct memory spaces. Everyone has direct access only to the local data Requires communication \\frametitle{High Performance Computing (memory models)} \\begin{center} \\includegraphics[width=0.6\\textwidth]{figures/memory.pdf} \\end{center} } \\frame{ \\frametitle{High Performance Computing (programming models)} \\begin{itemize} \\item The programming model changes when we aim for extra performance and/or memory: \\begin{enumerate} \\pause \\item {\\color{cyan} Single-core}: Matlab, Python, C, Fortran, \\dots \\begin{itemize} \\item Single stream of operations \\end{itemize} \\pause \\item {\\color{cyan} Multi-core}: Vectorized Matlab, pthreads, \\emph{OpenMP} \\begin{itemize} \\item \\emph{Multiple streams} of operations \\pause \\item \\emph{Work distribution}, \\emph{coordination} (synchronization, etc), \\dots \\end{itemize} \\pause \\item {\\color{cyan} Distributed memory}: \\emph{MPI}, \\dots \\begin{itemize} \\item Multiple streams of operations \\item Work distribution, coordination (synchronization, etc), \\dots \\pause \\item \\emph{Data distribution and communication} \\end{itemize} \\end{enumerate} \\pause \\item {\\color{cyan} GPUs}: \\emph{CUDA}, OpenCL, OpenACC, OpenMP, \\dots \\begin{itemize} \\pause \\item \\emph{Many lightweight} streams of operations \\pause \\item Work distribution, coordination (synchronization, etc), \\dots \\pause \\item \\emph{Data distribution across memory spaces and movement} \\end{itemize} \\end{itemize} } \\frame{ \\frametitle{High Performance Computing (software)} \\begin{itemize} \\item Complexity grows when we aim for extra performance and/or memory/storage: \\begin{enumerate} \\pause \\item {\\color{cyan} Single-core}: LAPACK, \\dots \\begin{itemize} \\item Load correct toolchain etc \\end{itemize} \\pause \\item {\\color{cyan} Multi-core}: LAPACK + parallel BLAS, \\dots \\begin{itemize} \\item Load correct toolchain etc \\pause \\item \\emph{Allocate} correct number of cores, \\emph{configure} software to use correct number of cores, \\dots \\end{itemize} \\pause \\item {\\color{cyan} Distributed memory}: ScaLAPACK, \\dots \\begin{itemize} \\item Load correct toolchain etc \\pause \\item Allocate correct number of \\emph{nodes and cores}, configure software to use correct number of \\emph{nodes and cores}, \\dots \\pause \\item Data distribution, storage, \\dots \\end{itemize} \\end{enumerate} \\pause \\item {\\color{cyan} GPUs}: MAGMA, TensorFlow, \\dots \\begin{itemize} \\item Load correct toolchain etc \\item Allocate correct number of \\emph{cores and GPUs}, configure software to use correct number of \\emph{cores and GPUs}, \\dots \\end{itemize} \\end{itemize} } \\frame{ \\frametitle{End (questions?)} \\centering \\Huge Questions? } \\end{document} Learning outcomes What is Linux? Learn about the command line interface (CLI) Navigate the file system - ls, mkdir, cd, rm, cp, mv, redirection, pipes, etc. Data handling - compressing, archiving, file transfer, patterns, etc. Hints and tricks of Linux","title":"High Performance Computing - memory models"},{"location":"intro/#why__linux","text":"Most HPC centers supercomputers run some flavour of Linux. It is stable and reliable Customizable Runs on any hardware Strong community Many flavours are open source and free Lots of applications Lightweight","title":"Why Linux"},{"location":"intro/#what__is__linux","text":"Linux is a family of open-source Unix-like operating systems based on the Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds. An operating system is the software that sits underneath all of the other software on a computer, managing the computer\u2019s hardware (CPU, GPU, memory, storage\u2026) and taking care of the connections between your other software and the hardware. Linux is typically packaged as a Linux distribution, which includes the kernel and supporting system software and libraries, many of which are provided by the GNU Project. Distributions (distros) There are many Linux distribuitions, including Ubuntu, Debian, Fedora, Gentoo, and many others. Many distributions are free and open source, but there are also commercial distributions, like Red Hat Enterprise and SUSE. Desktop Linux distributions include a desktop environment , like GNOME, MATE, KDE Plasma, Xfce, Unity, or many others. A window manager together with applications written using a widget toolkit are generally responsible for most of what the user sees. In addition, a windowing system of some sort (X11, Wayland) interfaces directly with the underlying operating system and libraries, providing support for graphical hardware, pointing devices, and keyboards. The window manager generally runs on top of this windowing system. While the use of Linux on desktop is only 2-3%, the vast majority of web servers (>96%), most mobile devices (Android is based on the Linux kernel), and all supercomputers on the Top500 list run Linux.","title":"What is Linux"},{"location":"intro/#more__information","text":"There is much more information about Linux on Wikipedia . Some pages with guides and/or cheat sheets: The Linux Handbook https://www.geeksforgeeks.org/linux-tutorial/ 14 Free Training Courses to Learn Linux Online Introduction to Linux - A Hands on Guide Linux Fundamentals Top 50+ Linux Commands You MUST Know","title":"More information"}]} \ No newline at end of file diff --git a/search/worker.js b/search/worker.js new file mode 100644 index 00000000..8628dbce --- /dev/null +++ b/search/worker.js @@ -0,0 +1,133 @@ +var base_path = 'function' === typeof importScripts ? '.' : '/search/'; +var allowSearch = false; +var index; +var documents = {}; +var lang = ['en']; +var data; + +function getScript(script, callback) { + console.log('Loading script: ' + script); + $.getScript(base_path + script).done(function () { + callback(); + }).fail(function (jqxhr, settings, exception) { + console.log('Error: ' + exception); + }); +} + +function getScriptsInOrder(scripts, callback) { + if (scripts.length === 0) { + callback(); + return; + } + getScript(scripts[0], function() { + getScriptsInOrder(scripts.slice(1), callback); + }); +} + +function loadScripts(urls, callback) { + if( 'function' === typeof importScripts ) { + importScripts.apply(null, urls); + callback(); + } else { + getScriptsInOrder(urls, callback); + } +} + +function onJSONLoaded () { + data = JSON.parse(this.responseText); + var scriptsToLoad = ['lunr.js']; + if (data.config && data.config.lang && data.config.lang.length) { + lang = data.config.lang; + } + if (lang.length > 1 || lang[0] !== "en") { + scriptsToLoad.push('lunr.stemmer.support.js'); + if (lang.length > 1) { + scriptsToLoad.push('lunr.multi.js'); + } + if (lang.includes("ja") || lang.includes("jp")) { + scriptsToLoad.push('tinyseg.js'); + } + for (var i=0; i < lang.length; i++) { + if (lang[i] != 'en') { + scriptsToLoad.push(['lunr', lang[i], 'js'].join('.')); + } + } + } + loadScripts(scriptsToLoad, onScriptsLoaded); +} + +function onScriptsLoaded () { + console.log('All search scripts loaded, building Lunr index...'); + if (data.config && data.config.separator && data.config.separator.length) { + lunr.tokenizer.separator = new RegExp(data.config.separator); + } + + if (data.index) { + index = lunr.Index.load(data.index); + data.docs.forEach(function (doc) { + documents[doc.location] = doc; + }); + console.log('Lunr pre-built index loaded, search ready'); + } else { + index = lunr(function () { + if (lang.length === 1 && lang[0] !== "en" && lunr[lang[0]]) { + this.use(lunr[lang[0]]); + } else if (lang.length > 1) { + this.use(lunr.multiLanguage.apply(null, lang)); // spread operator not supported in all browsers: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_operator#Browser_compatibility + } + this.field('title'); + this.field('text'); + this.ref('location'); + + for (var i=0; i < data.docs.length; i++) { + var doc = data.docs[i]; + this.add(doc); + documents[doc.location] = doc; + } + }); + console.log('Lunr index built, search ready'); + } + allowSearch = true; + postMessage({config: data.config}); + postMessage({allowSearch: allowSearch}); +} + +function init () { + var oReq = new XMLHttpRequest(); + oReq.addEventListener("load", onJSONLoaded); + var index_path = base_path + '/search_index.json'; + if( 'function' === typeof importScripts ){ + index_path = 'search_index.json'; + } + oReq.open("GET", index_path); + oReq.send(); +} + +function search (query) { + if (!allowSearch) { + console.error('Assets for search still loading'); + return; + } + + var resultDocuments = []; + var results = index.search(query); + for (var i=0; i < results.length; i++){ + var result = results[i]; + doc = documents[result.ref]; + doc.summary = doc.text.substring(0, 200); + resultDocuments.push(doc); + } + return resultDocuments; +} + +if( 'function' === typeof importScripts ) { + onmessage = function (e) { + if (e.data.init) { + init(); + } else if (e.data.query) { + postMessage({ results: search(e.data.query) }); + } else { + console.error("Worker - Unrecognized message: " + e); + } + }; +} diff --git a/sitemap.xml b/sitemap.xml new file mode 100644 index 00000000..d1120094 --- /dev/null +++ b/sitemap.xml @@ -0,0 +1,13 @@ + +