diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b4f6235..c6d1cb7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,6 +3,15 @@ name: Deploy Jupyter Book to GitHub Pages on: push: branches: [ master ] + paths-ignore: + - 'README.md' + - 'CONTRIBUTING.md' + - 'LICENSE' + pull_request: + paths-ignore: + - 'README.md' + - 'CONTRIBUTING.md' + - 'LICENSE' permissions: contents: read @@ -14,20 +23,13 @@ concurrency: cancel-in-progress: false jobs: - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} + build: runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Pages - uses: actions/configure-pages@v5 + - uses: actions/checkout@v6 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' @@ -35,13 +37,45 @@ jobs: run: pip install -r requirements.txt - name: Build Jupyter Book - run: jupyter-book build . + run: jupyter-book build . --warningiserror + + - name: Setup Pages + if: github.event_name == 'push' + uses: actions/configure-pages@v6 - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + if: github.event_name == 'push' + uses: actions/upload-pages-artifact@v5 with: path: _build/html + # Link checking runs independently and never blocks the build or deploy: + # external links can be flaky or rate-limit automated checkers. + linkcheck: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Check links + run: jupyter-book build . --builder linkcheck + + deploy: + needs: build + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 \ No newline at end of file + uses: actions/deploy-pages@v5 diff --git a/404.md b/404.md new file mode 100644 index 0000000..a95d6cd --- /dev/null +++ b/404.md @@ -0,0 +1,21 @@ +--- +orphan: true +myst: + html_meta: + description: "Page not found." + robots: "noindex" +--- + +# 404 — Page Not Found + +Sorry, the page you were looking for doesn't exist or may have moved. + +Try one of these instead: + +- [Home](index.md) +- [Worker](worker.md) +- [Applications](applications/index.md) +- [Blog](blog/index.md) + +If you followed a link here, please +[open an issue](https://github.com/open-lambda/open-lambda.github.io/issues) so we can fix it. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f77f7b5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,84 @@ +# Contributing to the OpenLambda Site + +This site is a [Jupyter Book](https://jupyterbook.org) (v1, Sphinx-based) written in +[MyST Markdown](https://myst-parser.readthedocs.io/). Pushing to `master` triggers the +GitHub Actions workflow in [`.github/workflows/deploy.yml`](.github/workflows/deploy.yml), +which builds the book and deploys it to GitHub Pages. + +## Local preview + +```bash +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +jupyter-book build . +open _build/html/index.html +``` + +```{note} +Keep dependencies pinned to the Jupyter Book **v1** stack in `requirements.txt`. Do **not** +bump `jupyter-book` to 2.x — v2 is a MyST-CLI rewrite that is incompatible with this repo's +`_config.yml` / `_toc.yml` and `sphinx_book_theme` setup. +``` + +## Adding a page + +1. Create a Markdown file (e.g. `applications/my-app.md`). +2. Register it in [`_toc.yml`](_toc.yml) so it appears in the navigation. +3. Add SEO metadata at the top of the file: + + ```yaml + --- + myst: + html_meta: + description: "One-sentence summary used as the meta description." + keywords: "OpenLambda, serverless, ..." + --- + ``` + +## Adding a blog post + +1. Create a file under `blog/post/` named `YYYY-MM-DD-short-slug.md`. The date prefix is the + convention used for ordering — it does not need to match the `date:` field exactly, but + keep them consistent. +2. Use this front matter (do **not** add blog posts to `_toc.yml` — ABlog collects them + automatically from the `blog/post/` directory): + + ```yaml + --- + blogpost: true + date: 2026-05-18 + author: jane-doe, john-smith + category: Case Studies # e.g. Releases, Case Studies + tags: openlambda, serverless # comma-separated + language: en + myst: + html_meta: + description: "One-sentence summary for search engines and link previews." + keywords: "OpenLambda, ..." + --- + + # Post Title + ``` + +3. The first `# Heading` is the post title; the first paragraph(s) become the excerpt shown + on the blog index. + +## Registering an author + +`author:` values are slugs. Map each slug to a display name and profile URL under +`blog_authors` in [`_config.yml`](_config.yml) so posts render a friendly name and link: + +```yaml +blog_authors: + jane-doe: + - "Jane Doe" + - "https://example.edu/jane" +``` + +An unregistered slug still works, but renders as the raw slug with no link. + +## Before opening a PR + +- Run `jupyter-book build .` locally and confirm it ends with `build succeeded` and no + warnings. +- Check that internal links resolve and external links are correct. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b2dee93 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative + Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 The OpenLambda Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 42a0660..24d6a41 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,86 @@ -# OpenLambda GitHub Pages — MyST Markdown Site +# OpenLambda Documentation Site -This repo deploys automatically to GitHub Pages using [Jupyter Book](https://jupyterbook.org) -with [MyST Markdown](https://myst-parser.readthedocs.io/) as the technical documentation of Open Lambda. +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Deploy](https://github.com/open-lambda/open-lambda.github.io/actions/workflows/deploy.yml/badge.svg)](https://github.com/open-lambda/open-lambda.github.io/actions/workflows/deploy.yml) + +Source for the **OpenLambda** documentation and blog, published at +****. + +The site is built with [Jupyter Book](https://jupyterbook.org) (v1, Sphinx-based) using +[MyST Markdown](https://myst-parser.readthedocs.io/) and the `sphinx_book_theme`, with +[ABlog](https://ablog.readthedocs.io/) powering the blog. Every push to `master` is built +and deployed to GitHub Pages automatically. + +> [!NOTE] +> This repo hosts the **website**, not the OpenLambda runtime. For the serverless platform +> itself, see [open-lambda/open-lambda](https://github.com/open-lambda/open-lambda). --- -## File Structure +## Repository layout ``` . -├── index.md ← Homepage content (MyST Markdown) -├── _config.yml ← Jupyter Book site settings -├── _toc.yml ← Table of contents (add more pages here) -├── requirements.txt ← Python deps for the build -├── .gitignore -└── .github/ - └── workflows/ - └── deploy.yml ← GitHub Actions auto-deploy workflow +├── index.md ← Homepage +├── worker.md ← Worker documentation +├── applications/ ← Application case studies +│ ├── index.md +│ └── ag-forecasting-api.md +├── blog/ ← Blog (ABlog) +│ ├── index.md +│ └── post/ ← One file per post: YYYY-MM-DD-slug.md +├── _config.yml ← Jupyter Book / Sphinx settings +├── _toc.yml ← Table of contents (register new pages here) +├── _static/ ← Logo, favicon, and other assets +├── requirements.txt ← Pinned Python build dependencies +├── CONTRIBUTING.md ← How to add pages, posts, and authors +├── LICENSE ← Apache 2.0 +└── .github/workflows/ + └── deploy.yml ← Build + deploy to GitHub Pages ``` --- -## One-Time Setup (do this once per repo) - -### 1. Clone the repo and copy these files in - -```bash -git clone https://github.com/open-lambda/open-lambda.github.io.git -cd open-lambda.github.io - -# Copy all files from this bundle into the repo root, then: -git add . -git commit -m "Add MyST Markdown Jupyter Book site" -git push -``` - -## Local Preview +## Local preview ```bash +python3 -m venv .venv && source .venv/bin/activate pip install -r requirements.txt jupyter-book build . -# Open in browser: -open _build/html/index.html +open _build/html/index.html # macOS; use xdg-open on Linux ``` + +> [!IMPORTANT] +> Dependencies are pinned to the Jupyter Book **v1** stack. Do **not** bump `jupyter-book` +> to 2.x — v2 is a MyST-CLI rewrite that is incompatible with this repo's `_config.yml`, +> `_toc.yml`, and `sphinx_book_theme` setup. + +A successful build ends with `build succeeded.` and writes HTML to `_build/html/`. + +--- + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for how to add a documentation page, write a blog +post (front matter, file naming, excerpts), and register a blog author. In short: + +- **New page** → add the `.md` file and register it in [`_toc.yml`](_toc.yml). +- **New blog post** → add `blog/post/YYYY-MM-DD-slug.md` with `blogpost: true` front + matter (ABlog collects it automatically — no `_toc.yml` entry needed). + +Please run `jupyter-book build .` locally and confirm it succeeds without warnings before +opening a pull request. + +--- + +## Deployment + +[`.github/workflows/deploy.yml`](.github/workflows/deploy.yml) builds the book and publishes +`_build/html` to GitHub Pages on every push to `master`. Changes to docs-only files +(`README.md`, `CONTRIBUTING.md`, `LICENSE`) do not trigger a deploy. + +--- + +## License + +Licensed under the **Apache License 2.0** — see [LICENSE](LICENSE) for the full text. diff --git a/_config.yml b/_config.yml index 60765dc..cafac1a 100644 --- a/_config.yml +++ b/_config.yml @@ -8,13 +8,13 @@ execute: execute_notebooks: "off" # Files Sphinx should ignore (keeps the GitHub README out of the toctree warning) -exclude_patterns: [README.md] +exclude_patterns: [README.md, CONTRIBUTING.md] # HTML-specific settings html: use_issues_button: true use_repository_button: true - use_edit_page_button: false + use_edit_page_button: true home_page_in_navbar: false extra_footer: |

Licensed under the Apache 2.0 License. @@ -48,13 +48,57 @@ parse: sphinx: extra_extensions: - ablog + - sphinxext.opengraph + - notfound.extension + - sphinx_sitemap config: html_theme: "sphinx_book_theme" html_show_sourcelink: false + html_favicon: "_static/open-lambda.png" + + # Canonical site URL — emits tags (SEO) and is + # required by sphinx_sitemap. + html_baseurl: "https://open-lambda.github.io/" + + # ---- Sitemap (sphinx_sitemap) ---- + # No language/version path segments on this site, so use a flat scheme. + sitemap_url_scheme: "{link}" + + # ---- Open Graph / social-card metadata ---- + ogp_site_url: "https://open-lambda.github.io/" + ogp_site_name: "OpenLambda" + ogp_image: "https://open-lambda.github.io/_static/open-lambda.png" + ogp_use_first_image: true + + # ---- Custom 404 page ---- + # This is a root (user/org) Pages site, so served from "/". + notfound_urls_prefix: "/" + + # ---- Link checking ---- + linkcheck_ignore: + # usenix.org returns 403 to automated checkers but works in-browser. + - "https://www\\.usenix\\.org/.*" + # GitHub rate-limits (429) the checker on nearly every run, which makes + # the result meaningless; GitHub links rarely rot, so skip them. + - "https://github\\.com/.*" + # Keep linkcheck fast: cap retries and how long we wait out a rate limit. + linkcheck_timeout: 15 + linkcheck_retries: 1 + linkcheck_rate_limit_timeout: 30 html_theme_options: + # Repository buttons. These must live here: a manually-defined + # html_theme_options block overrides the options Jupyter Book would + # otherwise generate from the top-level html:/repository: blocks. + repository_url: "https://github.com/open-lambda/open-lambda.github.io" + repository_branch: "master" + path_to_docs: "" + use_repository_button: true + use_issues_button: true + use_edit_page_button: true use_download_button: false navigation_depth: 2 show_navbar_depth: 1 + announcement: "🚧 OpenLambda is a research project under active development — follow along on GitHub." logo: text: "OpenLambda" @@ -68,10 +112,10 @@ sphinx: - "https://open-lambda.org" maria-oros: - "Maria Oros" - - "" + - "https://dsi.wisc.edu/staff/oros-maria/" tyler-caraza-harter: - "Tyler Caraza-Harter" - - "" + - "https://www.cs.wisc.edu/staff/harter-tyler-2/" blog_default_author: open-lambda blog_feed_archives: true post_date_format: "%Y-%m-%d" diff --git a/applications/ag-forecasting-api.md b/applications/ag-forecasting-api.md index c45c45d..f54edb1 100644 --- a/applications/ag-forecasting-api.md +++ b/applications/ag-forecasting-api.md @@ -1,11 +1,37 @@ +--- +myst: + html_meta: + description: "A FastAPI/ASGI application used as a case study for porting real workloads to OpenLambda." + keywords: "OpenLambda, serverless, FastAPI, ASGI, case study" +--- + # Agricultural Forecasting API -FastAPI-based ASGI service for crop disease risk forecasting using multi-source weather data, developed by the University of Wisconsin–Madison Data Science Institute and integrated with Open-Lambda technology. +A FastAPI-based ASGI service from the [UW–Madison Data Science Institute](https://dsi.wisc.edu/), +used here as a **case study for OpenLambda** — an existing, real-world application that was +never designed for serverless, ported to the platform to find out what OpenLambda needs. ---- +```{note} +This page focuses on what the port taught us about OpenLambda. For the application itself, +see its [source repository](https://github.com/UW-Madison-DSI/ag_forecasting_api). +``` + +## Why this application + +It's a good stress test for OpenLambda because it is non-trivial and ordinary: a FastAPI +(ASGI) backend with real package dependencies, weather-data caching, and parallel compute — +the kind of app teams actually run, not a serverless demo. + +## What it drove in OpenLambda + +Porting it surfaced five concrete challenges that drove four new OpenLambda features: + +- **Per-function environment variables** — to redirect writes away from read-only directories. +- **`pip-compile` as a lambda** — to pin dependencies inside the OpenLambda environment. +- **Direct GitHub deployment** — `ol admin install` straight from a repo URL. +- **Built-in ASGI support** — OpenLambda acts as the ASGI server for FastAPI/Starlette/etc. -## Overview +It also pointed to a future feature: `/dev/shm` support so process pools work under OpenLambda. -The API provides geospatial agricultural intelligence for Wisconsin, combining weather data with validated crop disease forecasting models. -The tool is now entering its next phase through integration with OpenLambda technology. -For reference, please check the documentation [here](https://github.com/UW-Madison-DSI/ag_forecasting_api.git). +The full porting write-up is in the +[blog case study](../blog/post/2026-05-18-ag-forecasting-case-study.md). diff --git a/applications/index.md b/applications/index.md index 5cf674f..b8c76b5 100644 --- a/applications/index.md +++ b/applications/index.md @@ -1,14 +1,42 @@ +--- +myst: + html_meta: + description: "Real-world applications ported to OpenLambda, and the platform features their needs drove." + keywords: "OpenLambda, serverless, applications, FastAPI, ASGI, WSGI" +--- + # Applications -We are building OL to support a wide range of applications with minimal changes required from those applications. Our approach is to port more real-world workloads and let their needs drive incremental improvements to the platform. +We are building OpenLambda to support a wide range of applications with minimal changes +required from those applications. Our approach is to port real-world workloads to the +platform and let their needs drive incremental improvements — rather than guessing which +features matter, we discover them by hitting real friction. + +## Why applications drive the roadmap + +Porting an application that was never designed for serverless surfaces concrete gaps. Each +gap becomes a candidate feature, and shipping that feature makes the next, similar +application easier to deploy. Recent work has been driven this way: + +- **WSGI and ASGI support.** OpenLambda removed its dependency on Tornado and added optional + support for both WSGI (the basis for Flask and Django) and ASGI (its async successor, used + by FastAPI and Starlette). ASGI enables higher concurrency and better I/O utilization, but + requires deeper runtime integration. +- **Lower onboarding friction.** Environment-variable configuration for functions and direct + GitHub-to-OpenLambda deployment reduce the steps needed to get a new workload running. +- **Practical runtime gaps.** Real apps surface constraints such as read-only directories and + the absence of `/dev/shm` for process pools — each a target for future work. + +A recurring design question is whether to consolidate multiple functions into a single +lambda-like unit. We believe co-locating functions is beneficial: it enables opportunistic +state reuse, faster warm starts, and reduced overhead. -## Want to deploy FastAPI apps on OpenLambda? Just ASGI us how. +Looking ahead, better visibility into execution — such as detecting when an application is +blocked on I/O — opens the door to billing models that distinguish active compute from idle +waiting, addressing a longstanding serverless concern. -In this post, we look at an agricultural forecasting application and an ASGI-based web service. The ag app is a natural fit for serverless because it is fundamentally stateless: results can be regenerated on demand from underlying data rather than relying on persistent runtime state. This also influenced the choice of FastAPI for a modern, async-friendly interface. -To situate ASGI, it builds on the long history of WSGI-based Python web serving, where earlier work (including Jaime’s contributions) helped shape synchronous server patterns. ASGI extends this model with native async support, enabling higher concurrency and better I/O utilization, but requiring deeper runtime integration. -A broader design question is whether it is beneficial to consolidate multiple functions into a single Lambda-like unit. We believe it is: co-locating functions enables opportunistic state reuse, faster warm starts, and reduced overhead. -On the implementation side, OL removes its dependency on Tornado and adds optional support for both WSGI and ASGI applications. The ag forecasting app also surfaced practical issues, including /dev/shm constraints when using process pools, which required targeted adjustments. -Looking ahead, improved visibility into execution—such as detecting when applications are blocked on I/O—opens the door to new billing models that distinguish compute from idle waiting time, addressing a longstanding serverless concern. -More broadly, expanding the set of supported applications continues to drive OL’s evolution, alongside features like environment variables and GitHub-based deployments that further reduce friction for onboarding new workloads. +## Case studies -- [Ag Forecasting API](ag-forecasting-api.md) \ No newline at end of file +- [**Ag Forecasting API**](ag-forecasting-api.md) — a FastAPI/ASGI crop-disease forecasting + service from the UW–Madison Data Science Institute. See the full porting write-up in the + [blog case study](../blog/post/2026-05-18-ag-forecasting-case-study.md). diff --git a/blog/index.md b/blog/index.md index d30a5f0..3660d88 100644 --- a/blog/index.md +++ b/blog/index.md @@ -1,3 +1,10 @@ +--- +myst: + html_meta: + description: "News, releases, and technical deep-dives from the OpenLambda serverless project." + keywords: "OpenLambda, blog, serverless, releases, case studies" +--- + # OpenLambda Blog News, releases, and technical deep-dives from the OpenLambda project. diff --git a/blog/post/2026-05-18-ag-forecasting-case-study.md b/blog/post/2026-05-18-ag-forecasting-case-study.md index fb36833..7b5352f 100644 --- a/blog/post/2026-05-18-ag-forecasting-case-study.md +++ b/blog/post/2026-05-18-ag-forecasting-case-study.md @@ -5,6 +5,10 @@ author: maria-oros, tyler-caraza-harter category: Case Studies tags: case-study, agforecast, asgi, fastapi, openlambda language: en +myst: + html_meta: + description: "Porting a FastAPI/ASGI application to OpenLambda — five challenges and the four platform features they drove." + keywords: "OpenLambda, serverless, FastAPI, ASGI, WSGI, pip-compile, case study" --- # An Application Case Study: Forecasting Crop Disease with OpenLambda @@ -20,13 +24,11 @@ In this post, we describe the challenges of porting AgForecast to OL, and four n ## Background: Agricultural Forecasting API -**Motivation.** The agricultural forecasting app is an open-source tool designed to address farmers' needs in a customizable way. We first built a backend infrastructure using FastAPI to serve crop disease forecasting models for corn and soybean, focusing on integration with Wisconet weather stations and on-demand model serving. We then developed a custom R Shiny front-end interface for farmer use. +**Motivation.** The agricultural forecasting app is an open-source tool designed to address farmers' needs in a customizable way providing daily crop disease forecasting predictions. We first built a backend infrastructure using FastAPI to serve crop disease forecasting models for multiple crops, focusing on integration with Wisconet weather stations and on-demand model serving. We then developed a custom python front-end interface for farmer use [here](https://github.com/UW-Madison-DSI/ag_forecasting_app_v3). -**Fit with OpenLambda.** We began exploring OpenLambda to leverage the benefits of serverless technology. Hosting our tool on this platform offers significant value not only to the developer community, but also to plant pathology practitioners and scientists who want to build on robust, nationwide infrastructure. +**Technical overview.** The Ag Forecasting API is a FastAPI-based backend that serves daily crop disease forecasting models across Wisconsin and plan to serve nationwide in the near future. It exposes two parallel data pipelines through a unified interface. The project ships with a `Dockerfile` and `docker-compose.yml` for containerized deployment, and includes a Starlette `WSGIMiddleware` wrapper so the FastAPI app can also be served behind WSGI servers for legacy or mixed environments. For more information, the API is MIT-licensed, fully open source, and currently deployed at [connect.doit.wisc.edu/ag_forecasting_api](https://connect.doit.wisc.edu/ag_forecasting_api). -**Technical overview.** The Ag Forecasting API is a FastAPI-based backend that serves crop disease forecasting models for corn and soybean across Wisconsin. It exposes two parallel data pipelines through a unified interface: one sourced from Wisconet, the public mesonet of weather stations across the state, and another from IBM Environmental Intelligence for point-location queries by latitude and longitude. The core logic lives in the `ag_models_wrappers` module, which dynamically pulls the daily and hourly weather variables each disease model requires for a given forecasting date, runs the risk calculations, and returns localized predictions. Currently supported models include Sporecaster (white mold in soybean, with dry and irrigated row-spacing variants), Tarspotter (tar spot of corn), gray leaf spot, and frogeye leaf spot — all based on peer-reviewed plant pathology research from UW–Madison. - -**Infrastructure and integration.** The project ships with a `Dockerfile` and `docker-compose.yml` for containerized deployment, and includes a Starlette `WSGIMiddleware` wrapper so the FastAPI app can also be served behind WSGI servers for legacy or mixed environments. A companion sub-package, `pywisconet`, provides a thin REST wrapper over the Wisconet v1 API with endpoints for active station discovery (`/wisconet/active_stations/`), station field metadata (`/station_fields/{station_id}`), and bulk measurement retrieval (`/bulk_measures/{station_id}`) at configurable frequencies (5-minute, hourly, or daily) for variables like air temperature, dew point, wind speed, and relative humidity. IBM credentials are handled via environment variables (`IBM_API_KEY`, `TENANT_ID`, `ORG_ID`) and validated against query parameters at request time. The API is MIT-licensed, fully open source, and currently deployed at [connect.doit.wisc.edu/ag_forecasting_api](https://connect.doit.wisc.edu/ag_forecasting_api). +**Fit with OpenLambda.** We began exploring OpenLambda to leverage the benefits of serverless technology. Hosting our tool on this platform offers significant value not only to the developer community, but also to plant pathology practitioners and scientists who want to keep expanding these innitiatives supported on this foundational robust infrastructure. ## Porting to OpenLambda @@ -138,4 +140,4 @@ We also made some minor changes to AgForecast: - customizable directory use for stations/measurements (challenge 1) - use of a thread pool instead of a process pool (challenge 5) -The second change suggests a future possible OL feature: adding `/dev/shm` availability to support process pools. \ No newline at end of file +The second change suggests a future possible OL feature: adding `/dev/shm` availability to support process pools. diff --git a/blog/post/2026-05-18-hello-world.md b/blog/post/2026-05-18-hello-world.md index 7a403e3..5418675 100644 --- a/blog/post/2026-05-18-hello-world.md +++ b/blog/post/2026-05-18-hello-world.md @@ -5,6 +5,10 @@ author: open-lambda category: Releases tags: announcement, openlambda language: en +myst: + html_meta: + description: "Welcome to the OpenLambda blog — release notes, performance experiments, and design discussions." + keywords: "OpenLambda, blog, announcement, serverless" --- # Hello, OpenLambda Blog diff --git a/index.md b/index.md index b878d5b..b2092ce 100644 --- a/index.md +++ b/index.md @@ -20,8 +20,8 @@ production as well. The main system implemented so far is a **single-node OpenLambda worker** that can take HTTP requests and invoke lambdas locally to compute responses. -You can read more about the OpenLambda worker [here](https://github.com/open-lambda/open-lambda/blob/main/docs/worker.md) -or just get started by [deploying a worker](https://github.com/open-lambda/open-lambda/blob/main/docs/quickstart.md). +You can read more about the OpenLambda worker on the [Worker page](worker.md), or just get +started by [deploying a worker](https://github.com/open-lambda/open-lambda/blob/main/docs/worker/getting-started.md). ```{note} We are currently working on a **cluster mode**, where a pool of VMs running the worker @@ -31,6 +31,36 @@ manually deploy workers yourself and put an HTTP load balancer in front of them. --- +## Explore + +::::{grid} 1 1 3 3 +:gutter: 3 + +:::{grid-item-card} 🛠️ The Worker +:link: worker +:link-type: doc + +The core server-side component — how it handles HTTP requests, manages containers, and scales horizontally. +::: + +:::{grid-item-card} 📦 Applications +:link: applications/index +:link-type: doc + +Real-world workloads ported to OpenLambda, and the platform features their needs drove. +::: + +:::{grid-item-card} 📰 Blog +:link: blog/index +:link-type: doc + +Release notes, performance experiments, and design deep-dives from the project. +::: + +:::: + +--- + ## Related Publications ```{list-table} diff --git a/requirements.txt b/requirements.txt index a332513..7e4c547 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,11 @@ -jupyter-book>=1.0.0 -sphinx-book-theme>=1.1.0 -myst-parser>=3.0.0 -sphinx-copybutton -ablog>=0.11 \ No newline at end of file +# Pinned to the Jupyter Book v1 (Sphinx-based) stack used by this site. +# Do NOT bump jupyter-book to 2.x: v2 is a MyST-CLI rewrite incompatible +# with this repo's _config.yml / _toc.yml and sphinx_book_theme setup. +jupyter-book==1.0.4.post1 +sphinx-book-theme==1.1.3 +myst-parser==3.0.1 +sphinx-copybutton==0.5.2 +ablog==0.11.13 +sphinxext-opengraph==0.9.1 +sphinx-notfound-page==1.0.4 +sphinx-sitemap==2.9.0 diff --git a/worker.md b/worker.md index ff2d221..cbd096c 100644 --- a/worker.md +++ b/worker.md @@ -1,62 +1,114 @@ +--- +myst: + html_meta: + description: "The OpenLambda worker — the core node component that runs lambdas in SOCK sandboxes and serves HTTP, Kafka, and cron triggers." + keywords: "OpenLambda, worker, serverless, SOCK, sandbox, HTTP, triggers, ol" +--- + # Worker The OpenLambda **worker** is the core server-side component of a node. It listens for -incoming HTTP requests, manages container lifecycle, and returns responses to callers. +incoming requests, runs lambdas inside isolated sandboxes, and returns responses to callers. +The worker is part of the `ol` binary. -## Overview +```{note} +This page is a high-level summary. The authoritative, version-tracked documentation lives in +the main repository under +[`docs/worker`](https://github.com/open-lambda/open-lambda/tree/main/docs/worker). +``` -Each worker is a standalone Go binary that exposes a single HTTP endpoint: +## Architecture -``` -POST /runLambda/ -``` +A worker is built from three layers, from the bottom up: -When a request arrives the worker: +1. **Sandbox** — isolates lambdas from one another. This layer is pluggable; the main + implementation today is **SOCK** (serverless-optimized containers), described in + [Oakes et al., ATC '18](https://www.usenix.org/conference/atc18/presentation/oakes). + Early versions used Docker. +2. **Lambda** — a *lambda instance* is a robust virtual container backed by zero or one + sandboxes; a *lambda function* routes incoming requests to instances and decides how many + instances to provision based on load. +3. **Event** — the trigger sources that invoke lambdas. -1. Checks whether the lambda's container image is already present on the node; if not, pulls it from the registry. -2. Starts a Linux container from the image. -3. Passes the request payload to the lambda function running inside the container. -4. Waits for the function to return a result, then forwards that result back to the caller. -5. Optionally keeps the container warm for a short period to reduce cold-start latency on subsequent calls. +## Triggers -## Configuration +The worker supports three kinds of triggers: -The worker is configured via a JSON file (default `config.json`) in the working directory. -Key fields: +- **HTTP** — a request to `http(s)://:/run/` invokes a lambda + directly. +- **Kafka** — a lambda can be configured to consume messages from a Kafka topic. +- **Cron** — a lambda can be invoked on a schedule. -| Field | Description | Default | -|---|---|---| -| `worker_port` | Port the HTTP server listens on | `8080` | -| `registry` | URL of the lambda registry | `""` | -| `sandbox` | Container backend (`docker` or `sock`) | `docker` | -| `log_output` | Where to write logs (`stdout` or a file path) | `stdout` | +## Building -## Starting the Worker +OpenLambda is actively tested on **Ubuntu 24.04 LTS**, requires **cgroups v2**, and relies on +operations that need root privilege. After installing the +dependencies listed in the +[getting-started guide](https://github.com/open-lambda/open-lambda/blob/main/docs/worker/getting-started.md), +build the Python-only ("min") deployment: ```bash -# From the repo root after building: -./bin/worker --config config.json +make ol imgs/ol-min ``` -The worker prints its listening address on startup. You can verify it is running with: +## Running a worker + +The `ol worker` subcommands manage a worker's lifecycle (run `./ol worker --help` for the +full list): ```bash -curl -w "\n" localhost:8080/status +# Create a worker directory with a config.json and base image +./ol worker init -i ol-min + +# Start the worker (use -d to run in the background) +./ol worker up -d + +# Check status +./ol worker status + +# Stop the worker +./ol worker down ``` -## Deploying Multiple Workers +`init` creates a worker directory (named `default-ol` by default) containing the worker +configuration (`config.json`), the read-only base image shared by all lambda instances, and +other resources. Per-worker settings such as memory limits are edited in `config.json`: + +```json +"limits": { + "mem_mb": 512 +} +``` + +## Deploying a lambda + +Functions can be installed directly from a Git repository, with dependencies pinned via a +`requirements.txt` and behavior configured via an `ol.yaml` file: + +```bash +./ol admin install -c ol.yaml -r requirements.txt https://github.com//.git + +# Invoke it +curl http://localhost:5000/run// +``` + +See the [Applications](applications/index.md) section and the +[Ag Forecasting case study](blog/post/2026-05-18-ag-forecasting-case-study.md) for a complete +worked example, including `ol.yaml` configuration and ASGI entry points. + +## Scaling out -Workers are stateless with respect to routing — each one operates independently. To scale -horizontally, start one worker process per node and place a standard HTTP load balancer -(Nginx, HAProxy, or similar) in front of them. No coordination between workers is required. +Workers are independent and require no coordination, so you can scale horizontally by running +one worker per node behind a standard HTTP load balancer (Nginx, HAProxy, or similar). ```{note} -A centralized **boss** component for cluster-wide management is currently under development. -Until then, manual deployment behind a load balancer is the recommended approach for -multi-node setups. +A centralized **boss** component for cluster-wide management is under development. Until then, +manual deployment behind a load balancer is the recommended approach for multi-node setups. ``` -## Further Reading +## Further reading -- [Quickstart guide](https://github.com/open-lambda/open-lambda/blob/main/docs/quickstart.md) — get a single worker running locally in minutes -- [SOCK: Rapid Task Provisioning with Serverless-Optimized Containers](https://www.usenix.org/conference/atc18/presentation/oakes) — the research paper describing the container backend. \ No newline at end of file +- [Getting started](https://github.com/open-lambda/open-lambda/blob/main/docs/worker/getting-started.md) — build, deploy, and run your first lambda +- [Lambda configuration](https://github.com/open-lambda/open-lambda/blob/main/docs/worker/lambda-config.md) — the `ol.yaml` format +- [Deploying applications](https://github.com/open-lambda/open-lambda/blob/main/docs/worker/apps.md) — example app walkthroughs +- [SOCK: Rapid Task Provisioning with Serverless-Optimized Containers](https://www.usenix.org/conference/atc18/presentation/oakes) — the sandbox research paper